From d31a6f628c341cc98fa180522abd817980627a18 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 01:49:49 +0200 Subject: [PATCH 01/38] deps: py3.12 floor + SACC/blinding stack (sacc, firecrown, smokescreen) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRD #241 row 1. sacc>=2.4 joins core dependencies (sp_validation.sacc_io will be core library code). firecrown v1.15.1 + smokescreen 1.5.6 + an exact pyccl pin form the new [blinding] extra: the blind must be exactly recomputable from the seed at unblinding time, so the theory stack is pinned as a set. firecrown is not on PyPI and hard-depends on numcosmo-py (conda-forge only) plus the cosmosis/cobaya sampler connectors we never import — the new uv-overrides.txt drops those three from resolution; the Dockerfile and README carry the --overrides invocation. Plain installs without the blinding extra are unaffected. requires-python moves 3.11 → 3.12: Smokescreen and firecrown both set a 3.12 floor, and the container base (shapepipe:develop) already runs python:3.12-slim-bookworm — this aligns pyproject with the actual runtime. ruff target-version follows. Validated: uv resolution of '.[test,glass,blinding]' with overrides (235 packages, py3.12) and the full firecrown-core + smokescreen import chain in a fresh venv on candide. The CI image build on this branch is the container-side proof. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019bGhVMAuhTy1gF6DdUc1dQ --- Dockerfile | 13 ++++++++----- README.md | 19 +++++++++++++++++++ pyproject.toml | 27 +++++++++++++++++++++++++-- uv-overrides.txt | 21 +++++++++++++++++++++ 4 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 uv-overrides.txt diff --git a/Dockerfile b/Dockerfile index c791877c..0ebb57a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,8 +28,11 @@ RUN uv pip install --no-cache-dir --upgrade 'cs_util>=0.2.1' WORKDIR /sp_validation COPY . /sp_validation -# Install with the test + glass extras so the image can run the unit suite in CI -# *and* the GLASS map-level mock test. `glass` (Generator for Large Scale -# Structure) ships `glass.ext.camb`; `cosmology` provides the `Cosmology` wrapper -# (`Cosmology.from_camb`) GLASS consumes. Both come in via the `[glass]` extra. -RUN uv pip install --no-cache-dir -e '.[test,glass]' +# Install with the test + glass + blinding extras so the image can run the unit +# suite in CI, the GLASS map-level mock test, *and* the SACC/Smokescreen blinding +# stack. `glass` (Generator for Large Scale Structure) ships `glass.ext.camb`; +# `cosmology` provides the `Cosmology` wrapper (`Cosmology.from_camb`) GLASS +# consumes. The `[blinding]` extra (firecrown + smokescreen) needs the override +# file: firecrown declares conda-forge-only / unused sampler connectors as hard +# deps — see uv-overrides.txt for the full story. +RUN uv pip install --no-cache-dir --overrides uv-overrides.txt -e '.[test,glass,blinding]' diff --git a/README.md b/README.md index e14a2071..640c6b20 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,25 @@ docker run --rm -it ghcr.io/cosmostat/sp_validation:develop python -c "import sp We do not currently build images for Apple Silicon/arm64; however the amd64 images should work on these systems, albeit with reduced performance. +## Local Installation + +Requires Python ≥ 3.12 (the floor is set by the blinding stack; the container +already runs 3.12). With [uv](https://docs.astral.sh/uv/): + +```bash +uv venv --python 3.12 +uv pip install -e '.[test]' +``` + +To also install the data-vector blinding stack (Smokescreen + firecrown, PRD +[#241](https://github.com/CosmoStat/sp_validation/issues/241)), pass the +dependency-override file — firecrown is not pip-resolvable without it (see +`uv-overrides.txt` for why): + +```bash +uv pip install --overrides uv-overrides.txt -e '.[test,blinding]' +``` + ## Flow chart diff --git a/pyproject.toml b/pyproject.toml index bb1fe413..b282d9ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,10 @@ authors = [ ] license = {text = "MIT"} readme = "README.md" -requires-python = ">=3.11" +# 3.12 floor set by Smokescreen 1.5.6 (and firecrown v1.15); the container base +# (shapepipe:develop) is already python:3.12-slim-bookworm, so this aligns +# pyproject with the actual runtime. +requires-python = ">=3.12" classifiers = [ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", @@ -52,6 +55,10 @@ dependencies = [ "pymaster", "regions", "reproject", + # SACC (LSST DESC's data-vector container) is the standard format for all + # data products from the tomographic round on (PRD #241); sp_validation.sacc_io + # is core library code, so sacc is a core dependency. + "sacc>=2.4,<3", # scipy 1.18 ported FITPACK from Fortran to C, changing the return shape of # RectBivariateSpline(scalar, scalar, grid=False) from 0-d `array(x)` to # shape-(1,) `array([x])`. camb's BBN Y_He predictor (bbn.py) wraps the @@ -106,6 +113,22 @@ glass = [ "glass.ext.camb==2023.6", "cosmology==2022.10.9", ] +# Data-vector blinding (PRD #241 §3-§5): Smokescreen applies the Muir et al. +# shift d → d + t(hidden) − t(fid), with firecrown + CCL as the theory engine +# (only compute_theory_vector is used; sampling stays with CosmoSIS). The blind +# must be exactly recomputable from the seed at unblinding time, so the whole +# theory stack is pinned exactly, as a set. Smokescreen 1.5.6 + firecrown v1.15 +# both set the python floor (>=3.12). +# +# firecrown is not on PyPI and declares conda-forge-only / unused sampler +# connectors as hard deps, so installing this extra requires the dependency +# override file: `uv pip install --overrides uv-overrides.txt -e '.[blinding]'` +# (see uv-overrides.txt; the Dockerfile does this for the container). +blinding = [ + "firecrown @ git+https://github.com/LSSTDESC/firecrown.git@v1.15.1", + "smokescreen==1.5.6", + "pyccl==3.3.4", +] develop = ["sp_validation[test,docs]"] [tool.pytest.ini_options] @@ -124,7 +147,7 @@ markers = [ [tool.ruff] line-length = 88 -target-version = "py311" +target-version = "py312" # Snakemake injects a `snakemake` object into rule scripts at runtime, so ruff # can't see where it's defined. Declaring it a builtin silences the false diff --git a/uv-overrides.txt b/uv-overrides.txt new file mode 100644 index 00000000..813ad055 --- /dev/null +++ b/uv-overrides.txt @@ -0,0 +1,21 @@ +# uv dependency overrides — pass via `--overrides uv-overrides.txt` (or +# UV_OVERRIDE=uv-overrides.txt) to every `uv pip install` against this project. +# +# Why this file exists: firecrown declares its sampler *connectors* as hard +# dependencies, but we use firecrown only as the theory engine for Smokescreen +# blinding (`compute_theory_vector`); sampling stays with CosmoSIS in +# cosmo_inference. Of the three connector deps: +# +# - numcosmo-py exists only on conda-forge, so pip/uv resolution of firecrown +# is *impossible* without an override; +# - cosmosis ships sdist-only (full Fortran/C build with gsl/cfitsio) — a +# heavy, fragile compile in every CI image build, for a connector we never +# import; +# - cobaya is wheel-clean but equally unused. +# +# Each line below replaces the package's requirement (wherever it appears in +# the graph) with one gated on an always-false marker, dropping it from +# resolution. firecrown's likelihood/CCL core imports none of them. +numcosmo-py; python_version < "3" +cosmosis; python_version < "3" +cobaya; python_version < "3" From 1d4826c8abb162a473d5761822361b4023f9f14b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:01:48 +0200 Subject: [PATCH 02/38] deps: firecrown pip-import fixes (numpy<2.5 cap, NumCosmo patch script) Follow-up hardening after empirical validation of the blinding stack on candide. Two facts surfaced that the resolve-only check couldn't see: - firecrown 1.15.1 subclasses npt.NDArray (DataVector); numpy 2.5 made that a non-subclassable typing alias, so firecrown breaks at import on the numpy>=2.0 resolution (2.5.1). The [blinding] extra now carries numpy>=2.2,<2.5 (2.4.3 verified against the full compiled stack + the fast suite), and the Dockerfile requests the bound explicitly to dodge uv #8410 non-movement of an already-installed numpy. - pip-installed firecrown hits NumCosmo (conda-forge-only) at import time through two paths unrelated to cosmic shear: eager re-export of LSST predefined n(z) bins in generators/__init__ (defeating upstream's own lazy __getattr__), and the cluster likelihoods -> lsstdesc-crow -> Ncm.IntegralND C-subclass at module load. scripts/patch_firecrown.py makes the bin re-export lazy, the cluster imports optional, and ships a loud numcosmo_py shim (raises on any real use). Exact-string surgery against pinned v1.15.1, idempotent, fails loudly on a version bump, ends with an import check. Verified: fresh application on a pristine install, idempotent re-run, and the full toy Smokescreen blind end-to-end after patching. CI now also smoke-tests the blinding imports in the built image - the fast suite never imports firecrown, so a broken blinding stack would otherwise ship green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019bGhVMAuhTy1gF6DdUc1dQ --- .github/workflows/deploy-image.yml | 6 + Dockerfile | 14 ++ README.md | 1 + pyproject.toml | 9 ++ scripts/patch_firecrown.py | 225 +++++++++++++++++++++++++++++ 5 files changed, 255 insertions(+) create mode 100644 scripts/patch_firecrown.py diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 192e87a6..13f0fafe 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -44,6 +44,12 @@ jobs: - name: Import smoke test run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sp_validation" + # The fast suite doesn't import the blinding stack, so a broken + # firecrown/smokescreen install would otherwise ship green. Prove the + # image can actually load it (sacc + patched firecrown + smokescreen). + - name: Blinding-stack import smoke test + run: docker run --rm ${{ steps.meta.outputs.tags }} python -c "import sacc; import firecrown.likelihood; import smokescreen" + # Run the fast test suite against the freshly-built image *before* # pushing, so a failing suite blocks publication. The image carries the # full stack and the test files (COPY . + editable install), so this diff --git a/Dockerfile b/Dockerfile index 0ebb57a8..818db67d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,3 +36,17 @@ COPY . /sp_validation # file: firecrown declares conda-forge-only / unused sampler connectors as hard # deps — see uv-overrides.txt for the full story. RUN uv pip install --no-cache-dir --overrides uv-overrides.txt -e '.[test,glass,blinding]' + +# Same uv gotcha as the cs_util upgrade above (astral-sh/uv #8410): if the base +# image already carries a numpy that violates the [blinding] extra's new +# `numpy<2.5` cap (firecrown 1.15.1 breaks on numpy 2.5 at import), the +# editable install won't move it. Request the bound explicitly so the image is +# deterministic either way; numpy 2.4.x is ABI-compatible with the compiled +# stack (verified: pyccl/camb/treecorr/healpy/pymaster + fast suite). +RUN uv pip install --no-cache-dir 'numpy>=2.2,<2.5' + +# firecrown is distributed for conda-forge (where NumCosmo always exists) and +# hits NumCosmo at import time in a pip env, on paths unrelated to our use. +# This patches the installed tree (surgical, pinned-version-checked, loud on +# mismatch) and verifies `import firecrown.likelihood; import smokescreen`. +RUN python scripts/patch_firecrown.py diff --git a/README.md b/README.md index 640c6b20..ef9e601b 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ dependency-override file — firecrown is not pip-resolvable without it (see ```bash uv pip install --overrides uv-overrides.txt -e '.[test,blinding]' +python scripts/patch_firecrown.py # make pip-installed firecrown importable without NumCosmo ``` diff --git a/pyproject.toml b/pyproject.toml index b282d9ad..ecdec6f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,10 +124,19 @@ glass = [ # connectors as hard deps, so installing this extra requires the dependency # override file: `uv pip install --overrides uv-overrides.txt -e '.[blinding]'` # (see uv-overrides.txt; the Dockerfile does this for the container). +# After installing this extra, run `python scripts/patch_firecrown.py` — it +# makes pip-installed firecrown importable without NumCosmo (conda-forge-only); +# see that script's docstring for the full story. blinding = [ "firecrown @ git+https://github.com/LSSTDESC/firecrown.git@v1.15.1", "smokescreen==1.5.6", "pyccl==3.3.4", + # firecrown 1.15.1 subclasses npt.NDArray (DataVector); numpy 2.5 turned + # npt.NDArray into a non-subclassable typing alias, breaking firecrown at + # import. firecrown's own env caps numpy<2.4; 2.4.3 is verified against + # the full compiled stack (pyccl/camb/treecorr/healpy/pymaster) + the + # sp_validation fast suite. + "numpy>=2.2,<2.5", ] develop = ["sp_validation[test,docs]"] diff --git a/scripts/patch_firecrown.py b/scripts/patch_firecrown.py new file mode 100644 index 00000000..7b528da1 --- /dev/null +++ b/scripts/patch_firecrown.py @@ -0,0 +1,225 @@ +"""Make pip-installed firecrown importable without NumCosmo. + +Run *inside* the target environment, after installing the ``[blinding]`` extra: + + python scripts/patch_firecrown.py + +Why this exists (PRD #241, PR 1): firecrown is the theory engine for +Smokescreen blinding — only ``compute_theory_vector`` on the SACC-read +cosmic-shear path is used. Upstream distributes firecrown via conda-forge, +where NumCosmo (a GObject-introspection C library, absent from PyPI) is always +present; in a pip/uv environment, firecrown 1.15.1 hits NumCosmo at *import +time* through two paths that have nothing to do with cosmic shear: + +1. ``firecrown/generators/__init__.py`` eagerly re-exports the LSST Y1/Y10 + predefined n(z) bin constants, defeating the lazy ``__getattr__`` that + ``_inferred_galaxy_zdist`` already provides — and computing those constants + imports NumCosmo. +2. ``firecrown/likelihood/__init__.py`` eagerly imports the cluster + likelihoods, which import ``crow`` (lsstdesc-crow), which subclasses a + NumCosmo C class at module load (``class CountsIntegralND(Ncm.IntegralND)``). + +This script (a) restores laziness in ``generators``, (b) makes the cluster +imports optional, and (c) installs a *loud* ``numcosmo_py`` shim so that any +genuine NumCosmo use raises immediately instead of being silently faked. +Everything is exact-string surgery against the pinned firecrown v1.15.1: if a +target string is missing (e.g. after a version bump), the script fails loudly +so the pin and the patch get reviewed together. Idempotent — safe to re-run. + +The right long-term fix is upstream (guarded/lazy imports in firecrown); until +then this file is the entire cost of staying pip-installable. +""" + +import importlib.metadata +import importlib.util +import subprocess +import sys +from pathlib import Path + +EXPECTED_FIRECROWN = "1.15.1" + +GENERATORS_OLD = """\ + # Lazy-loaded bins (via __getattr__) + Y1_LENS_BINS, + Y1_SOURCE_BINS, + Y10_LENS_BINS, + Y10_SOURCE_BINS, + LSST_Y1_LENS_HARMONIC_BIN_COLLECTION, + LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION, + LSST_Y10_LENS_HARMONIC_BIN_COLLECTION, + LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION, +) +""" + +GENERATORS_NEW = """\ +) + +# NOTE (sp_validation patch, scripts/patch_firecrown.py): the LSST Y1/Y10 +# predefined bin constants are computed lazily in _inferred_galaxy_zdist via a +# module-level __getattr__ that imports NumCosmo. Importing them EAGERLY here +# forced NumCosmo at `import firecrown.generators` (hence at +# `import firecrown.likelihood`), which pip cannot satisfy. Re-expose them +# lazily instead; the SACC-read cosmic-shear path never touches them. +_LAZY_BIN_NAMES = frozenset( + { + "Y1_LENS_BINS", + "Y1_SOURCE_BINS", + "Y10_LENS_BINS", + "Y10_SOURCE_BINS", + "LSST_Y1_LENS_HARMONIC_BIN_COLLECTION", + "LSST_Y1_SOURCE_HARMONIC_BIN_COLLECTION", + "LSST_Y10_LENS_HARMONIC_BIN_COLLECTION", + "LSST_Y10_SOURCE_HARMONIC_BIN_COLLECTION", + } +) + + +def __getattr__(name): + if name in _LAZY_BIN_NAMES: + from . import _inferred_galaxy_zdist as _z + + return getattr(_z, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + +""" + +LIKELIHOOD_OLD = """\ +# Cluster statistics +from firecrown.likelihood._binned_cluster import BinnedCluster +from firecrown.likelihood._binned_cluster_number_counts import ( + BinnedClusterNumberCounts, +) +from firecrown.likelihood._binned_cluster_number_counts_shear import ( + BinnedClusterShearProfile, +) +""" + +LIKELIHOOD_NEW = """\ +# Cluster statistics. +# NOTE (sp_validation patch, scripts/patch_firecrown.py): the cluster +# likelihoods import `crow` (lsstdesc-crow), which subclasses NumCosmo C +# classes at module load. NumCosmo is conda-forge-only, so in a pip/uv env +# these imports fail. They are NOT on the cosmic-shear (TwoPoint/WeakLensing) +# path, so they become optional: without NumCosmo the cluster classes are +# unavailable but everything else loads. +try: + from firecrown.likelihood._binned_cluster import BinnedCluster + from firecrown.likelihood._binned_cluster_number_counts import ( + BinnedClusterNumberCounts, + ) + from firecrown.likelihood._binned_cluster_number_counts_shear import ( + BinnedClusterShearProfile, + ) +except (ImportError, RuntimeError, TypeError): # pragma: no cover + BinnedCluster = None # type: ignore[assignment,misc] + BinnedClusterNumberCounts = None # type: ignore[assignment,misc] + BinnedClusterShearProfile = None # type: ignore[assignment,misc] +""" + +SHIM = '''\ +"""Minimal loud shim for numcosmo_py (installed by sp_validation). + +NumCosmo is a GObject-introspection C library available only via conda-forge. +With the companion patches to firecrown (scripts/patch_firecrown.py), the +SACC-read cosmic-shear likelihood path never imports it; this shim provides +the import-time names so the patched package loads, and any genuine numerical +use of NumCosmo raises loudly rather than being silently faked. +""" + + +class _Missing: + def __init__(self, path="numcosmo_py"): + self._p = path + + def __getattr__(self, name): + return _Missing(f"{self._p}.{name}") + + def __call__(self, *a, **k): + raise RuntimeError( + f"{self._p} was called, but NumCosmo is not installed (conda-forge " + "only, not on PyPI). It is not needed for the SACC-read " + "cosmic-shear likelihood path." + ) + + def __getitem__(self, item): + return _Missing(f"{self._p}[...]") + + +Ncm = _Missing("numcosmo_py.Ncm") +Nc = _Missing("numcosmo_py.Nc") +GObject = _Missing("numcosmo_py.GObject") + + +def dict_to_var_dict(*a, **k): + raise RuntimeError("numcosmo_py.dict_to_var_dict unavailable (no NumCosmo)") + + +def var_dict_to_dict(*a, **k): + raise RuntimeError("numcosmo_py.var_dict_to_dict unavailable (no NumCosmo)") +''' + + +def patch_file(path: Path, old: str, new: str) -> str: + text = path.read_text() + if new in text: + return "already patched" + if old not in text: + sys.exit( + f"FATAL: expected text not found in {path}.\n" + "firecrown has probably been bumped past the pinned version this " + "patch targets — review scripts/patch_firecrown.py together with " + "the [blinding] pin in pyproject.toml." + ) + path.write_text(text.replace(old, new, 1)) + return "patched" + + +def main() -> None: + spec = importlib.util.find_spec("firecrown") + if spec is None or spec.origin is None: + sys.exit("FATAL: firecrown is not installed in this environment.") + pkg = Path(spec.origin).parent + + # Metadata, not `import firecrown` — pre-patch, importing is what's broken. + version = importlib.metadata.version("firecrown") + if version != EXPECTED_FIRECROWN: + sys.exit( + f"FATAL: firecrown {version} != expected {EXPECTED_FIRECROWN}; " + "review this patch against the new version before bumping " + "EXPECTED_FIRECROWN." + ) + + print( + "generators/__init__.py:", + patch_file(pkg / "generators" / "__init__.py", GENERATORS_OLD, GENERATORS_NEW), + ) + print( + "likelihood/__init__.py:", + patch_file(pkg / "likelihood" / "__init__.py", LIKELIHOOD_OLD, LIKELIHOOD_NEW), + ) + + # Loud numcosmo_py shim — only when no real NumCosmo is present. + if importlib.util.find_spec("numcosmo_py") is None: + shim_dir = pkg.parent / "numcosmo_py" + shim_dir.mkdir(exist_ok=True) + (shim_dir / "__init__.py").write_text(SHIM) + print("numcosmo_py shim: installed") + else: + print("numcosmo_py shim: skipped (numcosmo_py importable)") + + check = subprocess.run( + [ + sys.executable, + "-c", + "import firecrown.likelihood; import smokescreen", + ], + capture_output=True, + text=True, + ) + if check.returncode != 0: + sys.exit(f"FATAL: post-patch import check failed:\n{check.stderr}") + print("post-patch import check: firecrown.likelihood + smokescreen OK") + + +if __name__ == "__main__": + main() From 6050be086a9f44194ddb1a4cc7a142f7be82ccf9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:36:19 +0200 Subject: [PATCH 03/38] feat(sacc_io): SACC read/write for the standard data-product layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sp_validation.sacc_io: the writer/reader layer for the two-file SACC layout that becomes the package's standard data-product format. {version}.sacc analysis vector — NZ tracers, coarse xi+/-, pseudo-Cl (EE/BB/EB) with a shared BandpowerWindow, COSEBIs, pure E/B, rho/tau PSF diagnostics; one FullCovariance assembled block-diagonally (zero cross-blocks). {version}_xi_fine COSEBIs/pure-EB integration input — same NZ tracers, fine-grid xi+/-, DiagonalCovariance from TreeCorr varxip/varxim. Covariance order is point-insertion order (SACC preserves it bitwise through FITS). Writers insert in the canonical order — xi+ then xi-, Cl (ee, bb, eb), COSEBIs (all En then all Bn), pure E/B in _EB_KEYS order (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, matching b_modes.calculate_eb_statistics), rho, then tau — but readers never assume global order: every getter resolves indices through s.indices(dtype, tracers, **tags). assemble_covariance validates that blocks are contiguous, ascending and tile the data vector exactly, failing loud otherwise. Custom data types (pure E/B, rho, tau) all parse under sacc.parse_data_type_name. Tag filters are plain kwargs; the tags={...} form silently selects nothing and is never used. Test suite (test_sacc_io.py, all synthetic and fast): per-writer round-trips (arrays/tags/windows/NZ bitwise), covariance block alignment and zero cross-blocks, assemble_covariance failure modes, DiagonalCovariance round-trip, extract() sub-covariance alignment, a tomographic multi-pair case, reader/writer mirroring on a mixed file, and the end-to-end two-file layout. 20 passed. Co-Authored-By: Claude Opus --- src/sp_validation/sacc_io.py | 537 ++++++++++++++++++++++++ src/sp_validation/tests/test_sacc_io.py | 491 ++++++++++++++++++++++ 2 files changed, 1028 insertions(+) create mode 100644 src/sp_validation/sacc_io.py create mode 100644 src/sp_validation/tests/test_sacc_io.py diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py new file mode 100644 index 00000000..cf03f12c --- /dev/null +++ b/src/sp_validation/sacc_io.py @@ -0,0 +1,537 @@ +"""SACC_IO. + +:Name: sacc_io.py + +:Description: Read/write the standard SACC data-product layout for the + weak-lensing validation package. Two files describe each + catalogue version: + + - ``{version}.sacc`` — the analysis vector: NZ tracers, coarse + ξ±, pseudo-Cℓ (EE/BB/EB) with bandpower windows, COSEBIs, + pure E/B, and ρ/τ PSF diagnostics, all sharing a single + ``FullCovariance`` assembled block-diagonally from the + per-statistic covariances (zero cross-blocks). + - ``{version}_xi_fine.sacc`` — the COSEBIs / pure-EB integration + input: the same NZ tracers, a fine-grid ξ±, and a + ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``. + + The covariance order is the point-insertion order (SACC preserves + it bitwise through FITS save/load). Writers below insert in the + canonical order — ξ+ then ξ−, Cℓ (ee, bb, eb), COSEBIs (all Eₙ + then all Bₙ), pure E/B (xip_E, xim_E, xip_B, xim_B, xip_amb, + xim_amb — matching ``b_modes._EB_KEYS``), ρ, then τ — but readers + never assume global order: they resolve indices through + ``Sacc.indices(dtype, tracers, **tags)``. + + Tag filters are plain keyword arguments to ``indices`` / + ``get_data_points`` / ``get_tag``; the ``tags={...}`` form + silently selects nothing and must never be used. +""" + +import numpy as np +import sacc + +PSF_TRACER = "psf_stars" + +# Standard SACC data-type strings. +XI_PLUS = "galaxy_shear_xi_plus" +XI_MINUS = "galaxy_shear_xi_minus" +CL_EE = "galaxy_shear_cl_ee" +CL_BB = "galaxy_shear_cl_bb" +CL_EB = "galaxy_shear_cl_eb" +COSEBI_EE = "galaxy_shear_cosebi_ee" +COSEBI_BB = "galaxy_shear_cosebi_bb" + +# Custom data-type strings (all parse under sacc.parse_data_type_name). +PURE_TYPES = { + "xip_E": "galaxy_shear_xiPureE_plus", + "xim_E": "galaxy_shear_xiPureE_minus", + "xip_B": "galaxy_shear_xiPureB_plus", + "xim_B": "galaxy_shear_xiPureB_minus", + "xip_amb": "galaxy_shear_xiPureAmb_plus", + "xim_amb": "galaxy_shear_xiPureAmb_minus", +} +# Insertion order of the six pure-EB blocks — matches b_modes._EB_KEYS, whose +# order is the [xip_E; xim_E; xip_B; xim_B; xip_amb; xim_amb] layout of the +# treecorr/MC pure-EB covariance (b_modes.calculate_eb_statistics, ~L392). +PURE_KEYS = ("xip_E", "xim_E", "xip_B", "xim_B", "xip_amb", "xim_amb") + +RHO_PLUS = "psf_rho{k}_xi_plus" +RHO_MINUS = "psf_rho{k}_xi_minus" +TAU_PLUS = "galaxyPsf_tau{k}_xi_plus" +TAU_MINUS = "galaxyPsf_tau{k}_xi_minus" + + +def source_name(i): + """SACC tracer name for source redshift bin ``i`` (0-based).""" + return f"source_{i}" + + +def new_sacc(nz, metadata=None): + """Create a Sacc with the survey's NZ (and PSF) tracers. + + Parameters + ---------- + nz : dict or sequence + Redshift distributions, one per source bin. Either a mapping + ``{i: (z, nz)}`` keyed by 0-based bin index, or a sequence of + ``(z, nz)`` array pairs (bin index = position). Tracers are named + ``source_{i}``. + metadata : dict, optional + Key/value pairs stored on ``s.metadata``. + + Returns + ------- + sacc.Sacc + Sacc holding the ``source_{i}`` NZ tracers and the ``psf_stars`` + Misc tracer (needed by ρ/τ diagnostics). + """ + items = nz.items() if isinstance(nz, dict) else enumerate(nz) + s = sacc.Sacc() + for i, (z, nz_i) in items: + s.add_tracer("NZ", source_name(i), np.asarray(z), np.asarray(nz_i)) + s.add_tracer("Misc", PSF_TRACER) + for key, value in (metadata or {}).items(): + s.metadata[key] = value + return s + + +def _pair(bins): + """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names.""" + i, j = bins + return (source_name(i), source_name(j)) + + +def add_xi( + s, + bins, + theta, + xip, + xim, + *, + grid, + theta_nom=None, + npairs=None, + weight=None, +): + """Add a real-space shear 2PCF (ξ+ then ξ−) for one tracer pair. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + theta : array_like + Angular separations (arcmin) — TreeCorr ``meanr``. + xip, xim : array_like + ξ+ and ξ− at ``theta``. + grid : {'coarse', 'fine'} + Distinguishes the analysis grid from the fine integration grid; + stored as the ``grid`` tag on every point. + theta_nom : array_like, optional + Nominal bin centres — TreeCorr ``rnom`` — stored as ``theta_nom``. + npairs, weight : array_like, optional + TreeCorr pair counts and weights, stored per point. + """ + tracers = _pair(bins) + for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): + for n, th in enumerate(theta): + tags = {"theta": float(th), "grid": grid} + if theta_nom is not None: + tags["theta_nom"] = float(theta_nom[n]) + if npairs is not None: + tags["npairs"] = float(npairs[n]) + if weight is not None: + tags["weight"] = float(weight[n]) + s.add_data_point(dtype, tracers, float(xi[n]), **tags) + + +def add_pseudo_cl( + s, + bins, + ell_eff, + cl_ee, + cl_bb, + cl_eb, + *, + window_ells, + window_weights, +): + """Add pseudo-Cℓ (EE, BB, EB) with a shared bandpower window. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + ell_eff : array_like + Effective multipole of each bandpower. + cl_ee, cl_bb, cl_eb : array_like + EE, BB and EB bandpowers at ``ell_eff``. + window_ells : array_like + Multipoles spanned by the bandpower window matrix (shape ``(nell,)``). + window_weights : array_like + Window matrix ``W`` of shape ``(nell, nbp)`` — one column per + bandpower — from NaMaster ``get_bandpower_windows``. One + ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. + """ + tracers = _pair(bins) + window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) + for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): + s.add_ell_cl( + dtype, *tracers, np.asarray(ell_eff), np.asarray(cl), window=window + ) + + +def add_cosebis(s, bins, En, Bn, scale_cut): + """Add COSEBIs (all Eₙ then all Bₙ) for one scale cut. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + En, Bn : array_like + E- and B-mode COSEBI amplitudes, one per logarithmic mode ``n`` + (1-based). The ``[En; Bn]`` layout matches the COSEBI covariance. + scale_cut : tuple of float + ``(theta_min, theta_max)`` in arcmin, stored on every point as the + ``theta_min``/``theta_max`` tags; multiple cuts coexist in one file, + told apart by these tags. + """ + tracers = _pair(bins) + theta_min, theta_max = scale_cut + for dtype, modes in ((COSEBI_EE, En), (COSEBI_BB, Bn)): + for n, value in enumerate(modes, start=1): + s.add_data_point( + dtype, + tracers, + float(value), + n=n, + theta_min=float(theta_min), + theta_max=float(theta_max), + ) + + +def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): + """Add pure E/B-mode correlation functions for one tracer pair. + + Six blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, + xim_B, xip_amb, xim_amb), matching ``b_modes._EB_KEYS`` and the pure-EB + covariance layout. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + theta : array_like + Angular separations (arcmin), shared by all six blocks. + xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like + The six pure E/B / ambiguous mode arrays at ``theta``. + """ + tracers = _pair(bins) + values = { + "xip_E": xip_E, + "xim_E": xim_E, + "xip_B": xip_B, + "xim_B": xim_B, + "xip_amb": xip_amb, + "xim_amb": xim_amb, + } + for key in PURE_KEYS: + dtype, arr = PURE_TYPES[key], values[key] + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def add_rho(s, k, theta, rho_p, rho_m): + """Add a ρ_k PSF statistic (ρ+ then ρ−) on the ``psf_stars`` tracer. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + k : int + ρ index (0…5). + theta : array_like + Angular separations (arcmin). + rho_p, rho_m : array_like + ρ_k+ and ρ_k− at ``theta``. + """ + tracers = (PSF_TRACER, PSF_TRACER) + for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def add_tau(s, bins, k, theta, tau_p, tau_m): + """Add a τ_k PSF-leakage statistic (τ+ then τ−). + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin ``i`` and PSF; the τ tracers are ``(source_i, psf_stars)``. + Only ``bins[0]`` is used. + k : int + τ index (0, 2 or 5). + theta : array_like + Angular separations (arcmin). + tau_p, tau_m : array_like + τ_k+ and τ_k− at ``theta``. + """ + tracers = (source_name(bins[0]), PSF_TRACER) + for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def assemble_covariance(s, blocks): + """Assemble a block-diagonal ``FullCovariance`` from per-statistic blocks. + + Each block is validated against the current insertion order: its indices + must be contiguous and ascending, the blocks must tile ``0…len(s.mean)`` + exactly (no gap, no overlap), and each block must be square with a size + matching its index span. Any violation raises ``ValueError`` naming the + mismatch. Cross-blocks are left zero. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place via ``add_covariance``. + blocks : sequence + Ordered ``(selector, cov)`` pairs (or a mapping of the same). Each + ``selector`` is either an index array, or a ``(data_type, tracers)`` + / ``(data_type, tracers, tags)`` tuple resolved through + ``s.indices``; ``cov`` is the block's dense covariance. + + Returns + ------- + sacc.Sacc + ``s``, with the assembled ``FullCovariance`` attached. + """ + items = blocks.items() if isinstance(blocks, dict) else blocks + ntot = len(s.mean) + full = np.zeros((ntot, ntot)) + cursor = 0 + for selector, cov in items: + idx = _resolve_indices(s, selector) + cov = np.asarray(cov) + if not np.array_equal(idx, np.arange(idx[0], idx[0] + len(idx))): + raise ValueError( + f"covariance block {selector!r} resolves to non-contiguous " + f"or non-ascending indices {idx.tolist()}" + ) + if idx[0] != cursor: + raise ValueError( + f"covariance block {selector!r} starts at index {idx[0]} but " + f"the previous blocks cover through {cursor} — blocks must tile " + "the data vector with no gap or overlap" + ) + if cov.ndim != 2 or cov.shape[0] != cov.shape[1]: + raise ValueError( + f"covariance block {selector!r} must be square; got shape {cov.shape}" + ) + if cov.shape[0] != len(idx): + raise ValueError( + f"covariance block {selector!r} has size {cov.shape[0]} but " + f"spans {len(idx)} data points" + ) + full[np.ix_(idx, idx)] = cov + cursor = idx[-1] + 1 + if cursor != ntot: + raise ValueError( + f"covariance blocks cover {cursor} of {ntot} data points — the " + "blocks must tile the whole data vector" + ) + s.add_covariance(full) + return s + + +def _resolve_indices(s, selector): + """Resolve a covariance-block selector to a sorted index array.""" + if isinstance(selector, (np.ndarray, list, tuple, range)) and not ( + len(selector) in (2, 3) and isinstance(selector[0], str) + ): + return np.asarray(selector, dtype=int) + data_type, tracers = selector[0], selector[1] + tags = selector[2] if len(selector) == 3 else {} + return np.asarray(s.indices(data_type, tuple(tracers), **tags), dtype=int) + + +def add_diagonal_covariance(s, variances): + """Attach a ``DiagonalCovariance`` from a 1-D variance array. + + The 1-D array is passed straight to ``add_covariance`` (never + ``np.diag``), which is what makes SACC store a ``DiagonalCovariance``. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + variances : array_like + Per-point variances, ``len == len(s.mean)``. + + Returns + ------- + sacc.Sacc + ``s``, with the ``DiagonalCovariance`` attached. + """ + s.add_covariance(np.asarray(variances)) + return s + + +def get_nz(s, i): + """Return ``(z, nz)`` for source bin ``i``.""" + tracer = s.tracers[source_name(i)] + return tracer.z, tracer.nz + + +def get_xi(s, bins, *, grid): + """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" + tracers = _pair(bins) + return ( + _sorted_tag(s, XI_PLUS, tracers, "theta", grid=grid), + _sorted_mean(s, XI_PLUS, tracers, grid=grid), + _sorted_mean(s, XI_MINUS, tracers, grid=grid), + ) + + +def get_pseudo_cl(s, bins): + """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. + + ``window`` is the shared ``sacc.BandpowerWindow`` recovered via + ``get_bandpower_windows``. + """ + tracers = _pair(bins) + ell = _sorted_tag(s, CL_EE, tracers, "ell") + window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) + return ( + ell, + _sorted_mean(s, CL_EE, tracers, _sort_tag="ell"), + _sorted_mean(s, CL_BB, tracers, _sort_tag="ell"), + _sorted_mean(s, CL_EB, tracers, _sort_tag="ell"), + window, + ) + + +def get_cosebis(s, bins, scale_cut=None): + """Return ``(n, En, Bn)`` for one tracer pair. + + Parameters + ---------- + scale_cut : tuple of float, optional + ``(theta_min, theta_max)`` to select when several cuts share the file. + """ + tracers = _pair(bins) + tags = ( + {"theta_min": float(scale_cut[0]), "theta_max": float(scale_cut[1])} + if scale_cut is not None + else {} + ) + modes = _sorted_tag(s, COSEBI_EE, tracers, "n", **tags) + return ( + modes.astype(int), + _sorted_mean(s, COSEBI_EE, tracers, **tags, _sort_tag="n"), + _sorted_mean(s, COSEBI_BB, tracers, **tags, _sort_tag="n"), + ) + + +def get_pure_eb(s, bins): + """Return ``(theta, {key: array})`` for the six pure-EB blocks. + + The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). + """ + tracers = _pair(bins) + theta = _sorted_tag(s, PURE_TYPES["xip_E"], tracers, "theta") + arrays = {key: _sorted_mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + return theta, arrays + + +def get_rho(s, k): + """Return ``(theta, rho_p, rho_m)`` for ρ index ``k``.""" + tracers = (PSF_TRACER, PSF_TRACER) + dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) + return ( + _sorted_tag(s, dt_p, tracers, "theta"), + _sorted_mean(s, dt_p, tracers), + _sorted_mean(s, dt_m, tracers), + ) + + +def get_tau(s, bins, k): + """Return ``(theta, tau_p, tau_m)`` for τ index ``k`` and source bin.""" + tracers = (source_name(bins[0]), PSF_TRACER) + dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) + return ( + _sorted_tag(s, dt_p, tracers, "theta"), + _sorted_mean(s, dt_p, tracers), + _sorted_mean(s, dt_m, tracers), + ) + + +def _order(s, data_type, tracers, sort_tag, **tag_filters): + """Indices for a selection, ordered by ascending ``sort_tag``.""" + idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) + key = np.array([s.data[i].tags[sort_tag] for i in idx]) + return idx[np.argsort(key)] + + +def _sorted_mean(s, data_type, tracers, _sort_tag="theta", **tag_filters): + """Mean values for a selection, ordered by ``_sort_tag`` ascending.""" + idx = _order(s, data_type, tracers, _sort_tag, **tag_filters) + return s.mean[idx] + + +def _sorted_tag(s, data_type, tracers, tag, **tag_filters): + """Values of ``tag`` for a selection, ordered by that tag ascending.""" + idx = _order(s, data_type, tracers, tag, **tag_filters) + return np.array([s.data[i].tags[tag] for i in idx]) + + +def extract(s, data_type=None, tracers=None, **tag_filters): + """Extract a sub-Sacc (points + aligned covariance sub-block). + + A copy is made and everything *not* matching the selection is removed, so + the covariance sub-block comes out correctly aligned and the original is + untouched. + + Parameters + ---------- + s : sacc.Sacc + Source, left unmodified. + data_type : str, optional + Data type to keep. + tracers : tuple, optional + Tracer pair to keep. + **tag_filters + Tag filters (plain kwargs, e.g. ``grid='fine'``). + + Returns + ------- + sacc.Sacc + New Sacc holding only the selected points. + """ + sub = s.copy() + selection = {} + if tracers is not None: + selection["tracers"] = tuple(tracers) + selection.update(tag_filters) + sub.keep_selection(data_type, **selection) + return sub + + +def save(s, path): + """Write ``s`` to ``path`` (FITS), overwriting any existing file.""" + s.save_fits(path, overwrite=True) + + +def load(path): + """Load a Sacc from ``path`` (FITS).""" + return sacc.Sacc.load_fits(path) diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py new file mode 100644 index 00000000..883a2af0 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_io.py @@ -0,0 +1,491 @@ +"""Tests for :mod:`sp_validation.sacc_io`. + +All synthetic, all fast: build in memory, round-trip through ``tmp_path``, +and assert arrays/tags/windows/covariances come back bitwise-identical and +correctly aligned. No cluster paths. +""" + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic builders +# --------------------------------------------------------------------------- # +def _nz(seed, n=50): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +def _theta(nbins=6): + return np.geomspace(1.0, 100.0, nbins) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n``.""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _base_sacc(nbins=1): + """A Sacc with ``nbins`` NZ source tracers plus the PSF tracer.""" + return sio.new_sacc({i: _nz(i) for i in range(nbins)}) + + +# --------------------------------------------------------------------------- # +# 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) +# --------------------------------------------------------------------------- # +def _roundtrip(s, tmp_path, name="rt"): + path = tmp_path / f"{name}.sacc" + sio.save(s, str(path)) + return sio.load(str(path)) + + +def test_nz_roundtrip(tmp_path): + z, nz = _nz(3) + s = sio.new_sacc({0: (z, nz)}, metadata={"version": "v1.4.6.3"}) + s2 = _roundtrip(s, tmp_path, "nz") + z2, nz2 = sio.get_nz(s2, 0) + assert np.array_equal(z2, z) + assert np.array_equal(nz2, nz) + assert s2.metadata["version"] == "v1.4.6.3" + assert sio.PSF_TRACER in s2.tracers + + +def test_xi_roundtrip(tmp_path): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + npairs, weight = np.arange(6) * 1e3, np.arange(6) * 1.5 + s = _base_sacc() + sio.add_xi( + s, + (0, 0), + theta, + xip, + xim, + grid="coarse", + theta_nom=theta * 1.01, + npairs=npairs, + weight=weight, + ) + s2 = _roundtrip(s, tmp_path, "xi") + th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) + assert np.array_equal(m, xim) + # extra tags survive + idx = s2.indices(sio.XI_PLUS, ("source_0", "source_0"), grid="coarse") + tags = s2.data[idx[0]].tags + assert tags["grid"] == "coarse" + assert set(tags) >= {"theta", "theta_nom", "npairs", "weight", "grid"} + + +def test_pseudo_cl_roundtrip(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0, 300.0]) + nell, nbp = 50, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + W = np.random.default_rng(5).uniform(size=(nell, nbp)) + ee, bb, eb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9, np.arange(nbp) * 3e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + bb, + eb, + window_ells=window_ells, + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "cl") + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + assert np.array_equal(cl_bb, bb) + assert np.array_equal(cl_eb, eb) + assert np.array_equal(window.weight, W) + assert np.array_equal(window.values, window_ells) + + +def test_cosebis_roundtrip(tmp_path): + En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + s = _base_sacc() + sio.add_cosebis(s, (0, 0), En, Bn, (1.0, 100.0)) + s2 = _roundtrip(s, tmp_path, "cosebi") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, En) + assert np.array_equal(B, Bn) + idx = s2.indices(sio.COSEBI_EE, ("source_0", "source_0")) + assert s2.data[idx[0]].tags["theta_min"] == 1.0 + assert s2.data[idx[0]].tags["theta_max"] == 100.0 + + +def test_pure_eb_roundtrip(tmp_path): + theta = _theta() + arrays = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + s = _base_sacc() + sio.add_pure_eb(s, (0, 0), theta, **arrays) + s2 = _roundtrip(s, tmp_path, "pureeb") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], arrays[key]) + + +def test_rho_roundtrip(tmp_path): + theta = _theta() + s = _base_sacc() + for k in range(6): + sio.add_rho( + s, k, theta, np.arange(6) * (k + 1) * 1e-6, np.arange(6) * (k + 1) * 2e-6 + ) + s2 = _roundtrip(s, tmp_path, "rho") + for k in range(6): + th, p, m = sio.get_rho(s2, k) + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-6) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-6) + + +def test_tau_roundtrip(tmp_path): + theta = _theta() + s = _base_sacc() + for k in (0, 2, 5): + sio.add_tau( + s, + (0, 0), + k, + theta, + np.arange(6) * (k + 1) * 1e-6, + np.arange(6) * (k + 1) * 2e-6, + ) + s2 = _roundtrip(s, tmp_path, "tau") + for k in (0, 2, 5): + th, p, m = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-6) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-6) + assert s2.data[s2.indices(sio.TAU_PLUS.format(k=k))[0]].tracers == ( + "source_0", + sio.PSF_TRACER, + ) + + +# --------------------------------------------------------------------------- # +# 2. Covariance alignment +# --------------------------------------------------------------------------- # +def _multi_statistic_sacc(): + """Sacc with ξ+/ξ−, Cℓ (ee/bb/eb) and COSEBIs, ready for a covariance.""" + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ell = np.array([30.0, 120.0, 210.0]) + W = np.random.default_rng(0).uniform(size=(20, 3)) + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + return s + + +def test_assemble_covariance_alignment(tmp_path): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + # Block selectors in canonical (insertion) order. + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov_xi, cov_cl, cov_co = _spd(len(xi), 1), _spd(len(cl), 2), _spd(len(co), 3) + sio.assemble_covariance(s, [(xi, cov_xi), (cl, cov_cl), (co, cov_co)]) + s2 = _roundtrip(s, tmp_path, "cov") + assert type(s2.covariance).__name__ == "FullCovariance" + dense = s2.covariance.dense + # each block's sub-covariance is exactly what went in + assert np.array_equal(dense[np.ix_(xi, xi)], cov_xi) + assert np.array_equal(dense[np.ix_(cl, cl)], cov_cl) + assert np.array_equal(dense[np.ix_(co, co)], cov_co) + # zero cross-blocks + assert np.array_equal(dense[np.ix_(xi, cl)], np.zeros((len(xi), len(cl)))) + assert np.array_equal(dense[np.ix_(xi, co)], np.zeros((len(xi), len(co)))) + assert np.array_equal(dense[np.ix_(cl, co)], np.zeros((len(cl), len(co)))) + + +def test_assemble_covariance_selector_tuples(): + """Blocks addressed by (data_type, tracers) tuples, not raw indices.""" + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + sio.assemble_covariance( + s, + [ + (xi, _spd(len(xi), 1)), + (cl, _spd(len(cl), 2)), + ((sio.COSEBI_EE, tr), _spd(len(s.indices(sio.COSEBI_EE, tr)), 3)), + ((sio.COSEBI_BB, tr), _spd(len(s.indices(sio.COSEBI_BB, tr)), 4)), + ], + ) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + +# --------------------------------------------------------------------------- # +# 3. assemble_covariance failure modes +# --------------------------------------------------------------------------- # +def test_assemble_covariance_wrong_dimension(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.arange(len(s.mean)) + with pytest.raises(ValueError, match="span"): + sio.assemble_covariance(s, [(idx, _spd(len(idx) - 1, 1))]) + + +def test_assemble_covariance_non_contiguous(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.array([0, 2, 4, 6, 8, 10, 1, 3]) # not contiguous/ascending + with pytest.raises(ValueError, match="non-contiguous"): + sio.assemble_covariance(s, [(idx, _spd(len(idx), 1))]) + + +def test_assemble_covariance_missing_coverage(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + with pytest.raises(ValueError, match="tile"): + sio.assemble_covariance( + s, [(xi, _spd(len(xi), 1))] + ) # leaves cl+cosebi uncovered + + +def test_assemble_covariance_non_square(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.arange(len(s.mean)) + with pytest.raises(ValueError, match="square"): + sio.assemble_covariance(s, [(idx, np.ones((len(idx), len(idx) - 1)))]) + + +def test_assemble_covariance_overlap(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + # second block starts before the first ended -> gap/overlap error + with pytest.raises(ValueError, match="tile|gap|overlap"): + sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (xi, _spd(len(xi), 2))]) + + +# --------------------------------------------------------------------------- # +# 4. Fine file: DiagonalCovariance from 1-D variances +# --------------------------------------------------------------------------- # +def test_diagonal_covariance_roundtrip(tmp_path): + theta = np.geomspace(0.1, 250.0, 40) + n = len(theta) + xip, xim = np.arange(n) * 1e-5, np.arange(n) * 2e-5 + varxip, varxim = np.arange(1, n + 1) * 1e-12, np.arange(1, n + 1) * 2e-12 + s = _base_sacc() + sio.add_xi(s, (0, 0), theta, xip, xim, grid="fine") + variances = np.concatenate([varxip, varxim]) # [xip; xim] order + sio.add_diagonal_covariance(s, variances) + assert type(s.covariance).__name__ == "DiagonalCovariance" + s2 = _roundtrip(s, tmp_path, "fine") + assert type(s2.covariance).__name__ == "DiagonalCovariance" + assert np.array_equal(np.diag(s2.covariance.dense), variances) + + +# --------------------------------------------------------------------------- # +# 5. extract(): sub-covariance alignment; original untouched; tag filter +# --------------------------------------------------------------------------- # +def test_extract_subblock_and_original_untouched(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov_co = _spd(len(co), 3) + sio.assemble_covariance( + s, [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, cov_co)] + ) + n_before = len(s.mean) + sub = sio.extract(s, data_type=sio.COSEBI_EE, tracers=tr) + # original untouched + assert len(s.mean) == n_before + # subset covariance equals the COSEBI-EE diagonal sub-block + ee_local = np.arange(len(s.indices(sio.COSEBI_EE, tr))) + assert np.array_equal(sub.covariance.dense, cov_co[np.ix_(ee_local, ee_local)]) + + +def test_extract_tag_filter(): + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + sio.add_xi(s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="fine") + sub = sio.extract( + s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="fine" + ) + assert len(sub.mean) == len(theta) + assert set(sub.get_tag("grid", sio.XI_PLUS)) == {"fine"} + + +# --------------------------------------------------------------------------- # +# 6. Tomographic case: >=2 bins, >=3 pairs, per-pair selection +# --------------------------------------------------------------------------- # +def test_tomographic_per_pair_selection(tmp_path): + theta = _theta() + s = _base_sacc(nbins=2) + pairs = [(0, 0), (0, 1), (1, 1)] + for k, (i, j) in enumerate(pairs): + sio.add_xi( + s, + (i, j), + theta, + np.arange(6) * (k + 1) * 1e-5, + np.arange(6) * (k + 1) * 2e-5, + grid="coarse", + ) + s2 = _roundtrip(s, tmp_path, "tomo") + for k, (i, j) in enumerate(pairs): + th, p, m = sio.get_xi(s2, (i, j), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-5) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-5) + # selecting one pair does not bleed into another + assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) + assert len(s2.indices(sio.XI_PLUS, ("source_1", "source_1"))) == len(theta) + + +# --------------------------------------------------------------------------- # +# 7. Readers mirror writers on a mixed file +# --------------------------------------------------------------------------- # +def test_readers_on_mixed_file(tmp_path): + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ell = np.array([30.0, 120.0, 210.0]) + W = np.random.default_rng(0).uniform(size=(20, 3)) + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + pure = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + sio.add_pure_eb(s, (0, 0), theta, **pure) + for k in range(6): + sio.add_rho( + s, k, theta, np.arange(6) * (k + 1) * 1e-7, np.arange(6) * (k + 1) * 2e-7 + ) + for k in (0, 2, 5): + sio.add_tau( + s, + (0, 0), + k, + theta, + np.arange(6) * (k + 1) * 1e-8, + np.arange(6) * (k + 1) * 2e-8, + ) + s2 = _roundtrip(s, tmp_path, "mixed") + + _, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(p, np.arange(6) * 1e-5) and np.array_equal( + m, np.arange(6) * 2e-5 + ) + ell_r, ee, bb, eb, win = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell) and np.array_equal(win.weight, W) + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(E, np.arange(1, 6) * 1e-6) + _, back = sio.get_pure_eb(s2, (0, 0)) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], pure[key]) + for k in range(6): + _, rp, rm = sio.get_rho(s2, k) + assert np.array_equal(rp, np.arange(6) * (k + 1) * 1e-7) + for k in (0, 2, 5): + _, tp, tm = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(tp, np.arange(6) * (k + 1) * 1e-8) + + +# --------------------------------------------------------------------------- # +# 8. End-to-end two-file layout for a synthetic catalogue version +# --------------------------------------------------------------------------- # +def test_end_to_end_two_file_layout(tmp_path): + version = "vSYNTH" + theta_c = _theta(20) + theta_f = np.geomspace(0.1, 250.0, 200) + + # analysis file + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="coarse" + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) + ) + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (co, _spd(len(co), 2))]) + sio.save(s, str(tmp_path / f"{version}.sacc")) + + # fine file + sf = _base_sacc() + sio.add_xi( + sf, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + ) + variances = np.concatenate([np.arange(1, 201) * 1e-12, np.arange(1, 201) * 2e-12]) + sio.add_diagonal_covariance(sf, variances) + sio.save(sf, str(tmp_path / f"{version}_xi_fine.sacc")) + + # reload both, verify everything + a = sio.load(str(tmp_path / f"{version}.sacc")) + f = sio.load(str(tmp_path / f"{version}_xi_fine.sacc")) + + th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="coarse") + assert np.array_equal(th_c, theta_c) and np.array_equal(p_c, np.arange(20) * 1e-5) + n, E, B = sio.get_cosebis(a, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert type(a.covariance).__name__ == "FullCovariance" + assert a.covariance.dense.shape == (len(a.mean), len(a.mean)) + + th_f, p_f, _ = sio.get_xi(f, (0, 0), grid="fine") + assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) + assert type(f.covariance).__name__ == "DiagonalCovariance" + assert np.array_equal(np.diag(f.covariance.dense), variances) From 66bbbc0e6ad27b901fbd3009132d011dd78d9fa2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:50:52 +0200 Subject: [PATCH 04/38] fix(sacc_io): enforce ascending grids; readers in insertion order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh-eyes review caught a correctness bug: readers re-sorted selections by theta/ell/n, but covariance blocks and bandpower windows stay in insertion order. On a non-ascending grid the reader output silently desynchronised from its covariance, and get_pseudo_cl returned sorted cl arrays against unsorted window columns — internally inconsistent within one return tuple. Fix by construction, not by sort: - Writers validate their grids. add_xi/add_pure_eb/add_rho/add_tau require strictly ascending theta; add_pseudo_cl requires strictly ascending ell_eff (add_cosebis is inherently safe — it enumerates the mode index). Out-of-order grids raise a loud ValueError naming the argument. - Readers drop the sort entirely and return in s.indices (insertion) order, so every getter is covariance- and window-aligned for ANY file, and ascending for canonical files. The _sorted_* helpers are replaced by plain insertion-order accessors (_mean/_tag). Also: - _pair normalises (i, j) -> sorted, so get_xi(s, (1, 0)) addresses the same symmetric shear-shear pair as (0, 1) instead of a silent empty read. - Module docstring documents the tomographic ξ covariance ordering: insertion is pair-major ([pair0 xip; pair0 xim; pair1 xip; …]), supplied to assemble_covariance as one contiguous block matching add_xi call order; type-major converters (DES 2pt-FITS) permute explicitly via s.indices. - extract() docstring states tracers takes SACC names, not integer bins. New tests (26 total, was 20): writers reject non-ascending theta/ell; (1, 0) == (0, 1) round-trip; a 3-pair tomographic ξ covariance assembled as one contiguous pair-major block with per-pair sub-blocks recovered via extract(); get_pseudo_cl window column j <-> ell_eff[j] via window_ind tags. Co-Authored-By: Claude Opus --- src/sp_validation/sacc_io.py | 114 +++++++++++++------ src/sp_validation/tests/test_sacc_io.py | 140 ++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 36 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index cf03f12c..4ed85d0f 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -26,6 +26,18 @@ Tag filters are plain keyword arguments to ``indices`` / ``get_data_points`` / ``get_tag``; the ``tags={...}`` form silently selects nothing and must never be used. + + Tomographic ξ ordering: each ``add_xi`` call inserts one tracer + pair as ``[xip; xim]``, so a multi-pair vector is *pair-major* + — ``[pair_0 xip; pair_0 xim; pair_1 xip; …]`` — not type-major + (``[all xip; all xim]``). A tomographic ξ covariance with + cross-pair correlations is therefore supplied to + ``assemble_covariance`` as ONE contiguous block spanning the + consecutive ``add_xi`` calls, ordered pair-by-pair to match + insertion. Writers must call ``add_xi`` in the same pair order + the covariance was built in. Converters that need a type-major + layout (e.g. the DES 2pt-FITS convention) permute explicitly via + ``s.indices`` rather than assuming global order. """ import numpy as np @@ -97,11 +109,32 @@ def new_sacc(nz, metadata=None): def _pair(bins): - """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names.""" - i, j = bins + """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names. + + The pair is normalised to ``i <= j``: shear-shear statistics are symmetric + in the tracer pair, and SACC stores each pair under one ordering, so + ``(1, 0)`` must address the same points as ``(0, 1)``. + """ + i, j = sorted(bins) return (source_name(i), source_name(j)) +def _check_ascending(name, values): + """Require ``values`` to be strictly ascending; else raise ValueError. + + Insertion order is the covariance (and bandpower-window) order, and readers + return points in insertion order, so an out-of-order grid would silently + desynchronise a data vector from its covariance. Enforce monotonicity at + write time instead. + """ + values = np.asarray(values) + if not np.all(np.diff(values) > 0): + raise ValueError( + f"{name} must be strictly ascending (insertion order is the " + f"covariance order); got {values.tolist()}" + ) + + def add_xi( s, bins, @@ -134,6 +167,7 @@ def add_xi( npairs, weight : array_like, optional TreeCorr pair counts and weights, stored per point. """ + _check_ascending("theta", theta) tracers = _pair(bins) for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): for n, th in enumerate(theta): @@ -177,6 +211,7 @@ def add_pseudo_cl( bandpower — from NaMaster ``get_bandpower_windows``. One ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. """ + _check_ascending("ell_eff", ell_eff) tracers = _pair(bins) window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): @@ -234,6 +269,7 @@ def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like The six pure E/B / ambiguous mode arrays at ``theta``. """ + _check_ascending("theta", theta) tracers = _pair(bins) values = { "xip_E": xip_E, @@ -263,6 +299,7 @@ def add_rho(s, k, theta, rho_p, rho_m): rho_p, rho_m : array_like ρ_k+ and ρ_k− at ``theta``. """ + _check_ascending("theta", theta) tracers = (PSF_TRACER, PSF_TRACER) for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): for n, th in enumerate(theta): @@ -286,6 +323,7 @@ def add_tau(s, bins, k, theta, tau_p, tau_m): tau_p, tau_m : array_like τ_k+ and τ_k− at ``theta``. """ + _check_ascending("theta", theta) tracers = (source_name(bins[0]), PSF_TRACER) for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): for n, th in enumerate(theta): @@ -397,9 +435,9 @@ def get_xi(s, bins, *, grid): """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" tracers = _pair(bins) return ( - _sorted_tag(s, XI_PLUS, tracers, "theta", grid=grid), - _sorted_mean(s, XI_PLUS, tracers, grid=grid), - _sorted_mean(s, XI_MINUS, tracers, grid=grid), + _tag(s, XI_PLUS, tracers, "theta", grid=grid), + _mean(s, XI_PLUS, tracers, grid=grid), + _mean(s, XI_MINUS, tracers, grid=grid), ) @@ -407,16 +445,17 @@ def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. ``window`` is the shared ``sacc.BandpowerWindow`` recovered via - ``get_bandpower_windows``. + ``get_bandpower_windows``; its columns are in the same insertion order as + the returned ``ell_eff``/``cl`` arrays, so window column ``j`` corresponds + to ``ell_eff[j]``. """ tracers = _pair(bins) - ell = _sorted_tag(s, CL_EE, tracers, "ell") window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) return ( - ell, - _sorted_mean(s, CL_EE, tracers, _sort_tag="ell"), - _sorted_mean(s, CL_BB, tracers, _sort_tag="ell"), - _sorted_mean(s, CL_EB, tracers, _sort_tag="ell"), + _tag(s, CL_EE, tracers, "ell"), + _mean(s, CL_EE, tracers), + _mean(s, CL_BB, tracers), + _mean(s, CL_EB, tracers), window, ) @@ -435,11 +474,11 @@ def get_cosebis(s, bins, scale_cut=None): if scale_cut is not None else {} ) - modes = _sorted_tag(s, COSEBI_EE, tracers, "n", **tags) + modes = _tag(s, COSEBI_EE, tracers, "n", **tags) return ( modes.astype(int), - _sorted_mean(s, COSEBI_EE, tracers, **tags, _sort_tag="n"), - _sorted_mean(s, COSEBI_BB, tracers, **tags, _sort_tag="n"), + _mean(s, COSEBI_EE, tracers, **tags), + _mean(s, COSEBI_BB, tracers, **tags), ) @@ -449,8 +488,8 @@ def get_pure_eb(s, bins): The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). """ tracers = _pair(bins) - theta = _sorted_tag(s, PURE_TYPES["xip_E"], tracers, "theta") - arrays = {key: _sorted_mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + theta = _tag(s, PURE_TYPES["xip_E"], tracers, "theta") + arrays = {key: _mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} return theta, arrays @@ -459,9 +498,9 @@ def get_rho(s, k): tracers = (PSF_TRACER, PSF_TRACER) dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) return ( - _sorted_tag(s, dt_p, tracers, "theta"), - _sorted_mean(s, dt_p, tracers), - _sorted_mean(s, dt_m, tracers), + _tag(s, dt_p, tracers, "theta"), + _mean(s, dt_p, tracers), + _mean(s, dt_m, tracers), ) @@ -470,28 +509,26 @@ def get_tau(s, bins, k): tracers = (source_name(bins[0]), PSF_TRACER) dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) return ( - _sorted_tag(s, dt_p, tracers, "theta"), - _sorted_mean(s, dt_p, tracers), - _sorted_mean(s, dt_m, tracers), + _tag(s, dt_p, tracers, "theta"), + _mean(s, dt_p, tracers), + _mean(s, dt_m, tracers), ) -def _order(s, data_type, tracers, sort_tag, **tag_filters): - """Indices for a selection, ordered by ascending ``sort_tag``.""" - idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) - key = np.array([s.data[i].tags[sort_tag] for i in idx]) - return idx[np.argsort(key)] +def _mean(s, data_type, tracers, **tag_filters): + """Mean values for a selection, in ``s.indices`` (insertion) order. - -def _sorted_mean(s, data_type, tracers, _sort_tag="theta", **tag_filters): - """Mean values for a selection, ordered by ``_sort_tag`` ascending.""" - idx = _order(s, data_type, tracers, _sort_tag, **tag_filters) - return s.mean[idx] + Never re-sort: insertion order is the covariance and bandpower-window + order, so returning in ``s.indices`` order keeps every reader aligned with + the covariance for any file (and ascending for canonically-written files, + which the writers enforce). + """ + return s.mean[s.indices(data_type, tracers, **tag_filters)] -def _sorted_tag(s, data_type, tracers, tag, **tag_filters): - """Values of ``tag`` for a selection, ordered by that tag ascending.""" - idx = _order(s, data_type, tracers, tag, **tag_filters) +def _tag(s, data_type, tracers, tag, **tag_filters): + """Values of ``tag`` for a selection, in insertion order.""" + idx = s.indices(data_type, tracers, **tag_filters) return np.array([s.data[i].tags[tag] for i in idx]) @@ -509,7 +546,12 @@ def extract(s, data_type=None, tracers=None, **tag_filters): data_type : str, optional Data type to keep. tracers : tuple, optional - Tracer pair to keep. + Tracer pair to keep, as SACC tracer **names** (e.g. + ``("source_0", "source_0")`` or ``("source_0", "psf_stars")``) — *not* + integer bin indices. This differs deliberately from the ``add_*`` / + ``get_*`` interface, whose ``bins`` argument takes integer pairs: + ``extract`` is the generic selection escape hatch, mirroring + ``Sacc.keep_selection`` and addressing non-source tracers uniformly. **tag_filters Tag filters (plain kwargs, e.g. ``grid='fine'``). diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 883a2af0..28bd6627 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -489,3 +489,143 @@ def test_end_to_end_two_file_layout(tmp_path): assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) assert type(f.covariance).__name__ == "DiagonalCovariance" assert np.array_equal(np.diag(f.covariance.dense), variances) + + +# --------------------------------------------------------------------------- # +# 9. Ascending-grid enforcement (writers reject out-of-order grids) +# --------------------------------------------------------------------------- # +def test_add_xi_rejects_non_ascending_theta(): + s = _base_sacc() + theta = _theta()[::-1] # descending + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + + +def test_add_pseudo_cl_rejects_non_ascending_ell(): + s = _base_sacc() + ell = np.array([210.0, 30.0, 120.0]) # not ascending + W = np.random.default_rng(0).uniform(size=(20, 3)) + with pytest.raises(ValueError, match="ell_eff must be strictly ascending"): + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + + +def test_add_pure_eb_rho_tau_reject_non_ascending_theta(): + s = _base_sacc() + theta = _theta()[::-1] + pure = {key: np.arange(6) * 1e-6 for key in sio.PURE_KEYS} + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_pure_eb(s, (0, 0), theta, **pure) + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_rho(s, 0, theta, np.arange(6) * 1e-6, np.arange(6) * 2e-6) + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_tau(s, (0, 0), 0, theta, np.arange(6) * 1e-6, np.arange(6) * 2e-6) + + +# --------------------------------------------------------------------------- # +# 10. Bin-pair normalisation: (1, 0) addresses the same points as (0, 1) +# --------------------------------------------------------------------------- # +def test_bin_pair_normalisation(): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + s = _base_sacc(nbins=2) + sio.add_xi(s, (0, 1), theta, xip, xim, grid="coarse") + th01, p01, m01 = sio.get_xi(s, (0, 1), grid="coarse") + th10, p10, m10 = sio.get_xi(s, (1, 0), grid="coarse") # reversed order + assert np.array_equal(th01, th10) + assert np.array_equal(p01, p10) and np.array_equal(p10, xip) + assert np.array_equal(m01, m10) and np.array_equal(m10, xim) + # writing under (1, 0) lands in the same tracer pair, not a new one + s2 = _base_sacc(nbins=2) + sio.add_xi(s2, (1, 0), theta, xip, xim, grid="coarse") + assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) + + +# --------------------------------------------------------------------------- # +# 11. Reader/covariance alignment holds for ANY insertion order (the core +# regression the review caught): a tomographic multi-pair ξ covariance +# assembled as ONE contiguous pair-major block, per-pair sub-blocks +# recovered via extract(). +# --------------------------------------------------------------------------- # +def test_tomographic_xi_covariance_one_contiguous_block(): + theta = _theta() + nth = len(theta) + pairs = [(0, 0), (0, 1), (1, 1)] + s = _base_sacc(nbins=2) + for k, (i, j) in enumerate(pairs): + sio.add_xi( + s, + (i, j), + theta, + np.arange(nth) * (k + 1) * 1e-5, + np.arange(nth) * (k + 1) * 2e-5, + grid="coarse", + ) + # All ξ points as one contiguous block in insertion (pair-major) order. + xi_idx = np.arange(len(s.mean)) + assert np.array_equal(xi_idx, np.arange(3 * 2 * nth)) # 3 pairs x [xip; xim] + cov = _spd(len(xi_idx), 11) # dense, cross-pair correlations + sio.assemble_covariance(s, [(xi_idx, cov)]) + + # Per-pair xip sub-block: resolve indices, extract, compare to input. + for i, j in pairs: + idx_p = s.indices( + sio.XI_PLUS, ("source_" + str(min(i, j)), "source_" + str(max(i, j))) + ) + sub = sio.extract( + s, + data_type=sio.XI_PLUS, + tracers=("source_" + str(min(i, j)), "source_" + str(max(i, j))), + ) + assert np.array_equal(sub.covariance.dense, cov[np.ix_(idx_p, idx_p)]) + # readers stay covariance-aligned: get_xi returns in the same order + th, xip, _ = sio.get_xi(s, (i, j), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(s.mean[idx_p], xip) + + +# --------------------------------------------------------------------------- # +# 12. get_pseudo_cl window/cl column correspondence: window column j maps to +# the returned ell_eff[j] (verified through window_ind tags). +# --------------------------------------------------------------------------- # +def test_pseudo_cl_window_column_correspondence(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0, 300.0]) + nell, nbp = 40, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + # Distinct columns so a permutation would be detectable. + W = np.zeros((nell, nbp)) + for b in range(nbp): + W[b * 5 : (b + 1) * 5, b] = 1.0 + ee, bb, eb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9, np.arange(nbp) * 3e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + bb, + eb, + window_ells=window_ells, + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "clwin") + ell, cl_ee, _, _, window = sio.get_pseudo_cl(s2, (0, 0)) + # returned ell array is in insertion order + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + # window_ind tag on each EE point indexes the matching window column + idx = s2.indices(sio.CL_EE, ("source_0", "source_0")) + for pos, i in enumerate(idx): + col = s2.data[i].tags["window_ind"] + assert col == pos # insertion order preserved => column j <-> ell[j] + assert np.array_equal(window.weight[:, col], W[:, pos]) From 2f4a8a883b02a89394811f2325c29fd5c6b0b68d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:19:29 +0200 Subject: [PATCH 05/38] feat(cosmo_val): SACC writers for the born-as-SACC data products MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sacc_writers.py: pure per-statistic *_to_sacc functions (xi coarse/fine, pseudo-Cl, COSEBIs, pure-E/B, rho/tau) that turn computed arrays into single-statistic SACC "parts", plus assemble_analysis_sacc, which rebuilds the single {version}.sacc analysis file with a FullCovariance assembled block- diagonally in canonical order (per the SACC layout contract — not concatenate_data_sets, whose BlockDiagonalCovariance the contract rules out). Add bandpower_window_from_workspace to pseudo_cl.py: the NEW plumbing that threads NaMaster's get_bandpower_windows() into sacc_io.add_pseudo_cl (the EE/BB/EB diagonal window is verified identical). Tests: 12 fast tests, each writer round-tripped and checked against the contract; a real small-nside NaMaster window round-trip; analysis assembly proven to cover every point with aligned blocks and zero cross-blocks, incl. the reload-from-disk DAG path. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/sacc_writers.py | 234 ++++++++++++++ src/sp_validation/pseudo_cl.py | 29 ++ src/sp_validation/tests/test_sacc_writers.py | 310 +++++++++++++++++++ 3 files changed, 573 insertions(+) create mode 100644 src/sp_validation/cosmo_val/sacc_writers.py create mode 100644 src/sp_validation/tests/test_sacc_writers.py diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py new file mode 100644 index 00000000..d227c379 --- /dev/null +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -0,0 +1,234 @@ +"""Born-as-SACC writers for the cosmo_val data products. + +A thin, pure layer between the ``cosmo_val`` mixins (which compute statistics as +TreeCorr / NaMaster / b_modes arrays) and :mod:`sp_validation.sacc_io` (which +knows the file layout). Each ``*_to_sacc`` function turns one already-computed +statistic into a single-statistic SACC — a *part* — carrying that statistic's +own covariance as its one covariance block. The Snakemake DAG writes one part +per rule; :func:`assemble_analysis_sacc` then loads the parts and rebuilds the +single ``{version}.sacc`` analysis file with a ``FullCovariance`` assembled +block-diagonally in canonical order (per the SACC layout contract — *not* +``sacc.concatenate_data_sets``, whose ``BlockDiagonalCovariance`` output the +contract rules out). + +The fine-grid ``{version}_xi_fine.sacc`` is a terminal product in its own right +(:func:`xi_to_sacc` with ``grid="fine"`` and a ``DiagonalCovariance`` from +TreeCorr ``varxip``/``varxim``); COSEBIs and pure-E/B consume it. + +Everything here is single-bin today (``bins=(0, 0)``); the interface is +tomography-native so a future round supplies real bin pairs unchanged. +""" + +import numpy as np + +from .. import sacc_io as sio +from ..pseudo_cl import bandpower_window_from_workspace + +# Statistics carried in the analysis file, and their custom-type k indices. +RHO_K = range(6) # ρ_0 … ρ_5 +TAU_K = (0, 2, 5) # τ_0, τ_2, τ_5 + +# NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). +_NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 + +BIN = (0, 0) # single-bin default until the round goes tomographic + + +def xi_to_sacc( + nz, + metadata, + theta, + xip, + xim, + *, + grid, + theta_nom=None, + npairs=None, + weight=None, + variances=None, +): + """One ξ± part (``bins=(0, 0)``) on the coarse or fine grid. + + ``variances`` (the concatenated ``[varxip; varxim]``) attaches a + ``DiagonalCovariance`` — used for the terminal fine file, where npatch=1 + leaves TreeCorr shot-noise variance as the only covariance estimate. + """ + s = sio.new_sacc(nz, metadata) + sio.add_xi( + s, + BIN, + theta, + xip, + xim, + grid=grid, + theta_nom=theta_nom, + npairs=npairs, + weight=weight, + ) + if variances is not None: + sio.add_diagonal_covariance(s, np.asarray(variances)) + return s + + +def pseudo_cl_to_sacc(nz, metadata, ell_eff, cl_all, wsp, covariance=None): + """One pseudo-Cℓ part: EE/BB/EB with the shared bandpower window. + + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); the + window comes from :func:`bandpower_window_from_workspace`. ``covariance``, + when given, is the dense ``[EE; BB; EB]``-ordered block matching insertion. + """ + window_ells, window_weights = bandpower_window_from_workspace(wsp) + s = sio.new_sacc(nz, metadata) + sio.add_pseudo_cl( + s, + BIN, + ell_eff, + cl_all[_NMT_EE], + cl_all[_NMT_BB], + cl_all[_NMT_EB], + window_ells=window_ells, + window_weights=window_weights, + ) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + return s + + +def cosebis_to_sacc(nz, metadata, result, scale_cut): + """One COSEBIs part at the fiducial scale cut. + + ``result`` is a single scale-cut result dict from + ``b_modes.calculate_cosebis`` — ``{"En", "Bn", "cov", ...}`` — where ``cov`` + is the ``[En; Bn]``-ordered COSEBIs covariance. Non-fiducial scale cuts are + a diagnostic (the PTE scan) and stay in the sidecar ``.npz``; only the + fiducial cut is a data product, because a ``FullCovariance`` must cover + every stored point and the cuts overlap in mode space. + """ + s = sio.new_sacc(nz, metadata) + sio.add_cosebis(s, BIN, result["En"], result["Bn"], scale_cut) + s.add_covariance(np.asarray(result["cov"])) + return s + + +def pure_eb_to_sacc(nz, metadata, theta, eb, covariance=None): + """One pure-E/B part: the six ``sacc_io.PURE_KEYS`` blocks. + + ``eb`` is a mapping with the six keys (``xip_E`` … ``xim_amb``); each array + is sampled at ``theta``. ``covariance``, when given, is the dense block in + ``PURE_KEYS`` order (matching ``b_modes._EB_KEYS`` and the insertion order). + """ + s = sio.new_sacc(nz, metadata) + sio.add_pure_eb(s, BIN, theta, **{key: eb[key] for key in sio.PURE_KEYS}) + if covariance is not None: + s.add_covariance(np.asarray(covariance)) + return s + + +def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov=None): + """One ρ/τ part: ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage. + + ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables + (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, ``rho_{k}_m``, … and + the τ analogue). ρ carries a diagonal (varxip/varxim) covariance — a + diagnostic placeholder, not used by inference — while τ carries ``tau_cov``, + the theoretical ``CovTauTh`` block the CosmoSIS τ-likelihood consumes. The + block order matches insertion: ρ (all +then−, per k) then τ. + """ + s = sio.new_sacc(nz, metadata) + theta_rho = np.asarray(rho_stats["theta"]) + for k in RHO_K: + sio.add_rho( + s, + k, + theta_rho, + np.asarray(rho_stats[f"rho_{k}_p"]), + np.asarray(rho_stats[f"rho_{k}_m"]), + ) + theta_tau = np.asarray(tau_stats["theta"]) + for k in TAU_K: + sio.add_tau( + s, + BIN, + k, + theta_tau, + np.asarray(tau_stats[f"tau_{k}_p"]), + np.asarray(tau_stats[f"tau_{k}_m"]), + ) + rho_var = np.concatenate( + [ + np.concatenate([rho_stats[f"varrho_{k}_p"], rho_stats[f"varrho_{k}_m"]]) + for k in RHO_K + ] + ) + tau_var = np.concatenate( + [ + np.concatenate([tau_stats[f"vartau_{k}_p"], tau_stats[f"vartau_{k}_m"]]) + for k in TAU_K + ] + ) + if tau_cov is None: + # Pure diagnostic file: diagonal covariance across ρ and τ. + s.add_covariance(np.concatenate([rho_var, tau_var])) + else: + # ρ diagonal (diagnostic) + τ dense theoretical block (inference input). + n_rho, n_tau = len(rho_var), len(tau_var) + tau_cov = np.asarray(tau_cov) + if tau_cov.shape != (n_tau, n_tau): + raise ValueError( + f"tau_cov shape {tau_cov.shape} does not match the {n_tau} τ " + "data points" + ) + full = np.zeros((n_rho + n_tau, n_rho + n_tau)) + full[:n_rho, :n_rho] = np.diag(rho_var) + full[n_rho:, n_rho:] = tau_cov + s.add_covariance(full) + return s + + +# --------------------------------------------------------------------------- # +# Analysis-file assembly +# --------------------------------------------------------------------------- # +def _copy_data_points(dst, src): + """Append every data point of ``src`` into ``dst`` (tags preserved).""" + for dp in src.data: + dst.add_data_point(dp.data_type, dp.tracers, dp.value, **dp.tags) + + +def assemble_analysis_sacc(nz, metadata, parts): + """Rebuild the single ``{version}.sacc`` analysis file from parts. + + Each part is a single-statistic Sacc (from a ``*_to_sacc`` writer, loaded + from disk) carrying its own covariance = its block. This re-adds every + part's data points into one Sacc in the order the parts are given — which + must be the canonical order (ξ± coarse, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ) + — and assembles a single ``FullCovariance`` from the per-part covariance + blocks. Point insertion order and block order therefore agree by + construction, which ``sacc_io.assemble_covariance`` validates (contiguous, + tiling, square) and raises on if they don't. + + Parameters + ---------- + nz, metadata : see :func:`sp_validation.sacc_io.new_sacc`. + parts : sequence of sacc.Sacc + Single-statistic parts, each with a covariance, in canonical order. + + Returns + ------- + sacc.Sacc + The analysis Sacc with a ``FullCovariance`` covering every point. + """ + s = sio.new_sacc(nz, metadata) + blocks = [] + cursor = 0 + for part in parts: + if part.covariance is None: + raise ValueError( + "every analysis part must carry its own covariance block; " + f"a part with data types {sorted(set(dp.data_type for dp in part.data))} " + "has none" + ) + n = len(part.mean) + _copy_data_points(s, part) + blocks.append((np.arange(cursor, cursor + n), part.covariance.dense)) + cursor += n + return sio.assemble_covariance(s, blocks) diff --git a/src/sp_validation/pseudo_cl.py b/src/sp_validation/pseudo_cl.py index c9355ec9..34cbc68d 100644 --- a/src/sp_validation/pseudo_cl.py +++ b/src/sp_validation/pseudo_cl.py @@ -280,3 +280,32 @@ def get_pseudo_cls_catalog( cl_all = wsp.decouple_cell(cl_coupled) return ell_eff, cl_all, wsp + + +# NaMaster spin-2 × spin-2 spectrum order: EE, EB, BE, BB. +_NMT_EE = 0 + + +def bandpower_window_from_workspace(wsp): + """Extract the bandpower window matrix ``W`` for a spin-2×spin-2 workspace. + + NaMaster's ``get_bandpower_windows()`` returns a four-index array + ``(n_cl_out, n_bpw, n_cl_in, n_ell)`` describing how each output bandpower + is built from the input multipoles across the EE/EB/BE/BB spectra. SACC's + ``BandpowerWindow`` model (one window per bandpower, shared across the + stored spectra) needs the per-spectrum *decoupling* window, i.e. the + diagonal EE←EE block (equal to BB←BB and EB←EB, verified identical). + + Returns + ------- + window_ells : np.ndarray + Multipoles the window spans, ``arange(n_ell)`` — the ``ell`` axis of + ``compute_coupled_cell``. + window_weights : np.ndarray + ``W`` of shape ``(n_ell, n_bpw)`` — one column per bandpower, the layout + :func:`sp_validation.sacc_io.add_pseudo_cl` expects. + """ + bpw = wsp.get_bandpower_windows() # (n_cl_out, n_bpw, n_cl_in, n_ell) + diagonal = bpw[_NMT_EE, :, _NMT_EE, :] # (n_bpw, n_ell) + window_ells = np.arange(diagonal.shape[1], dtype=float) + return window_ells, diagonal.T diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py new file mode 100644 index 00000000..057697e4 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -0,0 +1,310 @@ +"""Tests for :mod:`sp_validation.cosmo_val.sacc_writers`. + +Synthetic and fast: each ``*_to_sacc`` writer is exercised with in-memory +arrays, round-tripped through ``tmp_path``, and checked against the SACC layout +contract (data types, tags, ordering, covariance alignment). The analysis-file +assembler is verified to produce a single ``FullCovariance`` covering every +point with each per-statistic block correctly placed. One real small-nside +NaMaster round-trip proves the pseudo-Cℓ window survives the writer path. +""" + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _nz(seed=0, n=40): + rng = np.random.default_rng(seed) + return np.linspace(0.01, 2.0, n), rng.uniform(0.1, 1.0, n) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _theta(n=6): + return np.geomspace(1.0, 100.0, n) + + +def _roundtrip(s, tmp_path, name): + p = tmp_path / f"{name}.sacc" + sio.save(s, str(p)) + return sio.load(str(p)) + + +META = {"catalogue_version": "vSYNTH", "npatch": 1} + + +# --------------------------------------------------------------------------- # +# Per-writer parts +# --------------------------------------------------------------------------- # +def test_xi_to_sacc_coarse(tmp_path): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + s = sw.xi_to_sacc( + {0: _nz()}, META, theta, xip, xim, grid="coarse", theta_nom=theta * 1.01 + ) + s2 = _roundtrip(s, tmp_path, "xic") + th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) and np.array_equal(m, xim) + assert s2.covariance is None # coarse part has no cov until assembly + + +def test_xi_to_sacc_fine_diagonal(tmp_path): + theta = np.geomspace(0.5, 300.0, 30) + xip, xim = np.arange(30) * 1e-5, np.arange(30) * 2e-5 + varxip, varxim = np.arange(1, 31) * 1e-12, np.arange(1, 31) * 2e-12 + s = sw.xi_to_sacc( + {0: _nz()}, + META, + theta, + xip, + xim, + grid="fine", + variances=np.concatenate([varxip, varxim]), + ) + assert type(s.covariance).__name__ == "DiagonalCovariance" + s2 = _roundtrip(s, tmp_path, "xif") + th, p, _ = sio.get_xi(s2, (0, 0), grid="fine") + assert np.array_equal(th, theta) and np.array_equal(p, xip) + assert np.array_equal( + np.diag(s2.covariance.dense), np.concatenate([varxip, varxim]) + ) + + +def test_pseudo_cl_to_sacc_window_and_rows(tmp_path): + ell = np.array([30.0, 60.0, 90.0, 120.0]) + nbp = len(ell) + # NaMaster (4, nbp): EE, EB, BE, BB. + cl_all = np.vstack( + [ + np.arange(nbp) * 1e-9, + np.arange(nbp) * 2e-9, + np.zeros(nbp), + np.arange(nbp) * 3e-9, + ] + ) + + class _Wsp: + """Stand-in workspace: (n_cl_out, nbp, n_cl_in, nell) window array.""" + + def __init__(self, nbp, nell): + w = np.zeros((4, nbp, 4, nell)) + col = np.zeros((nbp, nell)) + for b in range(nbp): + col[b, b * 3 : b * 3 + 3] = 1.0 + for out in range(4): + w[out, :, out, :] = col + self._w = w + + def get_bandpower_windows(self): + return self._w + + s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell, cl_all, _Wsp(nbp, 24)) + s2 = _roundtrip(s, tmp_path, "cl") + ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell) + assert np.array_equal(ee, cl_all[0]) # EE row + assert np.array_equal(bb, cl_all[3]) # BB row (index 3, not 2=BE) + assert np.array_equal(eb, cl_all[1]) # EB row + assert window.weight.shape == (24, nbp) + + +def test_pseudo_cl_to_sacc_real_namaster(tmp_path): + """A real small-nside NaMaster workspace's window survives the writer.""" + pytest.importorskip("pymaster") + from sp_validation.pseudo_cl import get_pseudo_cls_map + + nside = 32 + mask = np.ones(12 * nside**2) + rng = np.random.default_rng(0) + shear = ( + rng.normal(size=12 * nside**2) + 1j * rng.normal(size=12 * nside**2) + ) * 1e-2 + ell_eff, cl_all, wsp = get_pseudo_cls_map(shear, mask, nside, "linear", ell_step=8) + s = sw.pseudo_cl_to_sacc({0: _nz()}, META, ell_eff, cl_all, wsp) + s2 = _roundtrip(s, tmp_path, "clreal") + ell_r, ee, bb, eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell_eff) + assert np.array_equal(ee, cl_all[0]) and np.array_equal(bb, cl_all[3]) + # window columns correspond to the bandpowers, one per ell_eff + assert window.weight.shape[1] == len(ell_eff) + + +def test_cosebis_to_sacc(tmp_path): + En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + result = {"En": En, "Bn": Bn, "cov": _spd(20, 7)} + s = sw.cosebis_to_sacc({0: _nz()}, META, result, (1.0, 100.0)) + s2 = _roundtrip(s, tmp_path, "co") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, En) and np.array_equal(B, Bn) + assert type(s2.covariance).__name__ == "FullCovariance" + assert np.array_equal(s2.covariance.dense, result["cov"]) + + +def test_pure_eb_to_sacc(tmp_path): + theta = _theta() + eb = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + cov = _spd(6 * len(theta), 9) + s = sw.pure_eb_to_sacc({0: _nz()}, META, theta, eb, covariance=cov) + s2 = _roundtrip(s, tmp_path, "eb") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], eb[key]) + assert np.array_equal(s2.covariance.dense, cov) + + +def _rho_tau_tables(nth=6, seed=0): + rng = np.random.default_rng(seed) + theta = _theta(nth) + rho = {"theta": theta} + for k in sw.RHO_K: + for suffix in ("p", "m"): + rho[f"rho_{k}_{suffix}"] = rng.normal(size=nth) * 1e-6 + rho[f"varrho_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, nth) + tau = {"theta": theta} + for k in sw.TAU_K: + for suffix in ("p", "m"): + tau[f"tau_{k}_{suffix}"] = rng.normal(size=nth) * 1e-6 + tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, nth) + return rho, tau, theta + + +def test_rho_tau_to_sacc_diagonal(tmp_path): + rho, tau, theta = _rho_tau_tables() + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau) + s2 = _roundtrip(s, tmp_path, "rt") + for k in sw.RHO_K: + th, p, m = sio.get_rho(s2, k) + assert np.array_equal(th, theta) + assert np.array_equal(p, rho[f"rho_{k}_p"]) + assert np.array_equal(m, rho[f"rho_{k}_m"]) + for k in sw.TAU_K: + th, p, m = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(p, tau[f"tau_{k}_p"]) + assert type(s2.covariance).__name__ == "DiagonalCovariance" + + +def test_rho_tau_to_sacc_tau_theory_block(tmp_path): + rho, tau, theta = _rho_tau_tables() + n_tau = 2 * len(sw.TAU_K) * len(theta) + tau_cov = _spd(n_tau, 11) + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov=tau_cov) + assert type(s.covariance).__name__ == "FullCovariance" + # τ sub-block equals the supplied theory covariance. + tr = ("source_0", sio.PSF_TRACER) + tau_idx = np.concatenate( + [ + np.concatenate( + [ + s.indices(sio.TAU_PLUS.format(k=k), tr), + s.indices(sio.TAU_MINUS.format(k=k), tr), + ] + ) + for k in sw.TAU_K + ] + ) + s2 = _roundtrip(s, tmp_path, "rttau") + assert np.allclose(s2.covariance.dense[np.ix_(tau_idx, tau_idx)], tau_cov) + + +def test_rho_tau_to_sacc_tau_cov_shape_mismatch(): + rho, tau, _ = _rho_tau_tables() + with pytest.raises(ValueError, match="tau_cov shape"): + sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov=_spd(3, 1)) + + +# --------------------------------------------------------------------------- # +# Analysis-file assembly +# --------------------------------------------------------------------------- # +def _make_parts(nz): + theta = _theta() + ell = np.array([30.0, 60.0, 90.0]) + + class _Wsp: + def get_bandpower_windows(self): + w = np.zeros((4, 3, 4, 20)) + for out in range(4): + for b in range(3): + w[out, b, out, b * 6 : b * 6 + 6] = 1.0 + return w + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + xi.add_covariance(_spd(len(xi.mean), 1)) + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + cl = sw.pseudo_cl_to_sacc(nz, META, ell, cl_all, _Wsp(), covariance=_spd(9, 2)) + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + return [xi, cl, co] + + +def test_assemble_analysis_sacc_full_covariance(tmp_path): + nz = {0: _nz()} + parts = _make_parts(nz) + s = sw.assemble_analysis_sacc(nz, META, parts) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + # every point covered; blocks placed and cross-blocks zero + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl_idx = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co_idx = np.concatenate( + [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] + ) + assert len(xi_idx) + len(cl_idx) + len(co_idx) == len(s.mean) + dense = s.covariance.dense + assert np.array_equal(dense[np.ix_(xi_idx, xi_idx)], parts[0].covariance.dense) + assert np.array_equal(dense[np.ix_(cl_idx, cl_idx)], parts[1].covariance.dense) + assert np.array_equal(dense[np.ix_(co_idx, co_idx)], parts[2].covariance.dense) + assert np.array_equal( + dense[np.ix_(xi_idx, cl_idx)], np.zeros((len(xi_idx), len(cl_idx))) + ) + # round-trips + s2 = _roundtrip(s, tmp_path, "analysis") + assert type(s2.covariance).__name__ == "FullCovariance" + assert np.allclose(s2.covariance.dense, s.covariance.dense) + + +def test_assemble_analysis_sacc_requires_covariance(): + nz = {0: _nz()} + parts = _make_parts(nz) + parts.append( + sw.xi_to_sacc( + nz, META, _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ) # no covariance + with pytest.raises(ValueError, match="own covariance block"): + sw.assemble_analysis_sacc(nz, META, parts) + + +def test_assemble_from_reloaded_parts(tmp_path): + """Parts written to disk then reloaded assemble identically (the DAG path).""" + nz = {0: _nz()} + parts = _make_parts(nz) + reloaded = [] + for i, part in enumerate(parts): + sio.save(part, str(tmp_path / f"part{i}.sacc")) + reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) + s = sw.assemble_analysis_sacc(nz, META, reloaded) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) From 18ec7ff94cfdb751301d00eb5980c3936ee2324b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:25:37 +0200 Subject: [PATCH 06/38] =?UTF-8?q?feat(twopoint):=20SACC=20=E2=86=92=202pt-?= =?UTF-8?q?FITS=20converter,=20byte-compatible=20with=20cosmosis=5Ffitting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sp_validation.twopoint_convert.sacc_to_twopoint_fits, converting an analysis SACC into the CosmoSIS 2pt-FITS that 2pt_like (and Sacha's rho/tau 2pt_like_xi_sys fork) reads. Output reproduces today's cosmo_inference cosmosis_fitting.py assembly HDU-for-HDU and byte-for-byte: NZDATA, XI_PLUS/MINUS, CELL_EE, the blocked COVMAT (STRT_i offsets) and separate COVMAT_CELL, and the TAU_{0,2}_PLUS + verbatim RHO_STATS tables. The data vector and covariance are permuted from SACC's pair-major order to the 2pt-FITS type-major layout via s.indices. The tau covariance is laid in as one contiguous [tau_0+; tau_2+] block, preserving the tau_0<->tau_2 cross-correlation that covdat_to_fits carries. Rho/tau HDUs need the rho/tau sidecar FITS: the RHO_STATS varrho_* variance columns are not stored in the analysis SACC, so the converter copies them verbatim (never fabricates them); xi, Cl, n(z) and the covariance are fully reconstructed from SACC alone. Tests: - test_twopoint_convert.py: byte-equal vs current cosmosis_fitting.py builders on deterministic synthetic inputs for all three product shapes (plain xi; xi+Cl; xi+rho/tau), plus the tau_0<->tau_2 cross-correlation and perturbation teeth. - test_twopoint_convert_realdata.py (skipif on candide paths): build a SACC from a real product's own contents, convert, byte-compare vs the current writer. Observed byte-equal for SP_v1.4.6_leak_corr and a glass_mock sibling; the stale on-disk files differ only by an extra CELL_BB HDU (older script version) and ~1e-17 float noise on bin edges, documented and guarded. Co-Authored-By: Claude Fable 5 --- .../tests/test_twopoint_convert.py | 378 ++++++++++++++++++ .../tests/test_twopoint_convert_realdata.py | 268 +++++++++++++ src/sp_validation/twopoint_convert.py | 330 +++++++++++++++ 3 files changed, 976 insertions(+) create mode 100644 src/sp_validation/tests/test_twopoint_convert.py create mode 100644 src/sp_validation/tests/test_twopoint_convert_realdata.py create mode 100644 src/sp_validation/twopoint_convert.py diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py new file mode 100644 index 00000000..5e599717 --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -0,0 +1,378 @@ +"""Byte-compare tests for the SACC -> 2pt-FITS converter. + +The converter (:mod:`sp_validation.twopoint_convert`) must reproduce the CosmoSIS +2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, +so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau +``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check +is *byte* equality, and astropy writes FITS deterministically, so that is what we +assert: build a reference with the current script's own HDU-builder functions on +deterministic synthetic inputs, build a SACC from those same inputs via +:mod:`sp_validation.sacc_io`, convert it, and compare the two files byte for byte. + +Three configurations pin the three product shapes today's ``__main__`` emits: + +1. **plain xi** -- PRIMARY, NZ_SOURCE, COVMAT, XI_PLUS, XI_MINUS. +2. **xi + pseudo-Cl** -- adds COVMAT_CELL and CELL_EE (the harmonic block + ``2pt_like`` reads; the script builds CELL_BB and discards it, so the + converter does too). +3. **xi + rho/tau** -- adds the blocked tau covariance (TAU_0_PLUS / TAU_2_PLUS, + with the tau_0<->tau_2 cross-correlation the truncated CosmoCov tau + covariance carries) and the verbatim RHO_STATS table. + +The rho/tau product needs the rho/tau *sidecar* HDUs: the RHO_STATS table carries +per-mode ``varrho_*`` variances the analysis SACC does not store, so the +converter copies them from the sidecar exactly as today's assembly does. A teeth +test pins that a perturbed input moves the output. + +The reference builder imports ``cosmosis_fitting.py`` by path (it is a script, +not a package module), skipping cleanly if a dependency is missing -- the same +loader pattern as ``test_cosmosis_fitting.py``. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + """Import cosmosis_fitting.py by path; skip cleanly if a dep is missing.""" + if not _SCRIPT.exists(): + pytest.skip(f"cosmosis_fitting.py not found at {_SCRIPT}") + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except ImportError as exc: # pragma: no cover - container has numpy/astropy + pytest.importorskip(getattr(exc, "name", "") or "cosmosis_fitting_dependency") + raise + return module + + +cf = _load_cf() + + +# --- deterministic synthetic inputs ----------------------------------------- +# +# One tomographic bin (today's analysis). N_ANG angular bins on an ascending +# theta grid; N_ELL bandpowers; a 200-point n(z) on a uniform z grid (the DES +# NZDATA table needs a uniform Z_MID axis). The xi covariance is a full +# (2*N_ANG) matrix (xi+/xi- cross-block nonzero, as CosmoCov produces); the tau +# covariance is a full (3*N_ANG) matrix that the assembly truncates to its first +# 2 statistics (tau_0, tau_2). + +N_ANG = 5 +N_ELL = 8 +SOURCE = sacc_io.source_name(0) +PSF = sacc_io.PSF_TRACER + + +def _spd(n, seed): + """A symmetric positive-definite (n, n) matrix, seeded and recognizable.""" + rng = np.random.default_rng(seed) + a = rng.normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _inputs(seed=0): + """Deterministic synthetic statistics + covariances for one bin pair.""" + rng = np.random.default_rng(seed) + theta = np.sort(rng.uniform(1.0, 250.0, N_ANG)) + ell = np.sort(rng.uniform(30.0, 3000.0, N_ELL)) + z = np.linspace(0.0125, 0.4875, 200) + return { + "theta": theta, + "ell": ell, + "z": z, + "nz": np.exp(-((z - 0.25) ** 2) / 0.02), + "xip": rng.uniform(1e-6, 1e-4, N_ANG), + "xim": rng.uniform(1e-6, 1e-4, N_ANG), + "cl_ee": rng.uniform(1e-10, 1e-8, N_ELL), + "cl_bb": rng.uniform(1e-11, 1e-9, N_ELL), + "cl_eb": rng.uniform(-1e-11, 1e-11, N_ELL), + "tau0p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau0m": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2m": rng.uniform(1e-6, 1e-5, N_ANG), + "xi_cov": _spd(2 * N_ANG, seed + 1), + "cl_cov": _spd(N_ELL, seed + 2), + "tau_cov_full": _spd(3 * N_ANG, seed + 3), + } + + +# --- reference sidecar files + HDUs (the current script's own builders) ------ + + +def _rho_sidecar_hdu(theta, seed=7): + """A rho-stats BinTableHDU with the 25-column layout (values + variances).""" + rng = np.random.default_rng(seed) + columns = [fits.Column(name="theta", format="D", array=theta)] + for k in range(6): + for suffix in ("_p", "_m"): + columns.append( + fits.Column( + name=f"rho_{k}{suffix}", + format="D", + array=rng.uniform(-1e-3, 1e-3, len(theta)), + ) + ) + columns.append( + fits.Column( + name=f"varrho_{k}{suffix}", + format="D", + array=rng.uniform(1e-15, 1e-12, len(theta)), + ) + ) + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _tau_sidecar_hdu(theta, tau0p, tau2p): + """A tau-stats BinTableHDU with theta + tau_0_p + tau_2_p columns.""" + columns = [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _reference_fits(tmp_path, inp, *, cl=False, rho_tau=False): + """Build the reference 2pt-FITS with cosmosis_fitting.py's own functions. + + Reproduces the exact ``__main__`` HDU list for the requested configuration, + which is the assembly the converter must match byte for byte. + """ + nz_txt = tmp_path / "nz.txt" + np.savetxt(nz_txt, np.column_stack([inp["z"], inp["nz"]])) + cov_txt = tmp_path / "cov_xi.txt" + np.savetxt(cov_txt, inp["xi_cov"]) + + nz_hdu = cf.nz_to_fits(str(nz_txt)) + xip_hdu = cf._create_2pt_hdu(inp["xip"], inp["theta"], "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(inp["xim"], inp["theta"], "XI_MINUS", "G-R", "G-R") + + hdu_list = [fits.PrimaryHDU(), nz_hdu] + + if rho_tau: + tau_cov_npy = tmp_path / "cov_tau.npy" + np.save(tau_cov_npy, inp["tau_cov_full"]) + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=str(tau_cov_npy)) + else: + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=None) + hdu_list.append(cov_hdu) + + if cl: + cl_block = np.zeros((5, N_ELL)) + cl_block[0], cl_block[1], cl_block[4] = inp["ell"], inp["cl_ee"], inp["cl_bb"] + cl_npy = tmp_path / "cl.npy" + np.save(cl_npy, cl_block) + cl_cov_npy = tmp_path / "cl_cov.npy" + np.save(cl_cov_npy, inp["cl_cov"]) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(cl_npy)) + cl_ee_hdu, _cl_bb_hdu = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(cl_cov_npy), cov_hdu="COVAR_FULL") + hdu_list.append(cov_cl_hdu) + + hdu_list.extend([xip_hdu, xim_hdu]) + if cl: + hdu_list.append(cl_ee_hdu) + + if rho_tau: + rho_path = tmp_path / "rho.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"]), + ] + ).writeto(tau_path, overwrite=True) + rho_hdu = cf.rho_to_fits(str(rho_path), theta=inp["theta"]) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tau_path), theta=inp["theta"]) + hdu_list.extend([tau0_hdu, tau2_hdu, rho_hdu]) + + out = tmp_path / "reference.fits" + fits.HDUList(hdu_list).writeto(out, overwrite=True) + return out + + +# --- the SACC each configuration is built from ------------------------------ + + +def _sacc(inp, *, cl=False, rho_tau=False): + """Build the analysis SACC the converter reads, matching ``_inputs``. + + The covariance is laid out to match the reference exactly: the xi block is + the full (2*N_ANG) matrix; the Cl block is the EE bandpower covariance; the + tau blocks carry the tau_0<->tau_2 cross-correlation from the truncated + CosmoCov tau covariance. Blocks not consumed by the 2pt-FITS (Cl BB/EB, tau + minus) get an identity block so ``add_covariance`` sees a full matrix. + """ + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_xi(s, (0, 0), inp["theta"], inp["xip"], inp["xim"], grid="coarse") + if cl: + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + if rho_tau: + sacc_io.add_tau(s, (0, 0), 0, inp["theta"], inp["tau0p"], inp["tau0m"]) + sacc_io.add_tau(s, (0, 0), 2, inp["theta"], inp["tau2p"], inp["tau2m"]) + + n = len(s.mean) + full = np.zeros((n, n)) + ip = s.indices(sacc_io.XI_PLUS, (SOURCE, SOURCE)) + im = s.indices(sacc_io.XI_MINUS, (SOURCE, SOURCE)) + xi_all = np.concatenate([ip, im]) + full[np.ix_(xi_all, xi_all)] = inp["xi_cov"] + + if cl: + iee = s.indices(sacc_io.CL_EE, (SOURCE, SOURCE)) + full[np.ix_(iee, iee)] = inp["cl_cov"] + for dtype in (sacc_io.CL_BB, sacc_io.CL_EB): + idx = s.indices(dtype, (SOURCE, SOURCE)) + full[np.ix_(idx, idx)] = np.eye(N_ELL) + if rho_tau: + t0p = s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)) + t2p = s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)) + tau_pp = np.concatenate([t0p, t2p]) + # The joint [tau_0+; tau_2+] block is the truncated CosmoCov tau + # covariance -- cross-correlation kept, matching covdat_to_fits. + full[np.ix_(tau_pp, tau_pp)] = inp["tau_cov_full"][: 2 * N_ANG, : 2 * N_ANG] + for dtype in (sacc_io.TAU_MINUS.format(k=0), sacc_io.TAU_MINUS.format(k=2)): + idx = s.indices(dtype, (SOURCE, PSF)) + full[np.ix_(idx, idx)] = np.eye(N_ANG) + s.add_covariance(full) + return s + + +def _sidecar_hdus(tmp_path, inp): + """Return the (rho_hdu, tau_hdu) sidecar input HDUs for the rho/tau product.""" + rho_path = tmp_path / "rho_in.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau_in.fits" + fits.HDUList( + [fits.PrimaryHDU(), _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"])] + ).writeto(tau_path, overwrite=True) + with fits.open(rho_path) as r, fits.open(tau_path) as t: + return r[1].copy(), t[1].copy() + + +# ============================================================================= +# Byte-compare: the three product shapes +# ============================================================================= + + +def test_plain_xi_byte_equal(tmp_path): + """Plain-xi product: converter matches cosmosis_fitting.py byte for byte.""" + inp = _inputs(seed=0) + reference = _reference_fits(tmp_path, inp) + s = _sacc(inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_cl_byte_equal(tmp_path): + """xi + pseudo-Cl product: COVMAT_CELL + CELL_EE reproduced byte for byte.""" + inp = _inputs(seed=10) + reference = _reference_fits(tmp_path, inp, cl=True) + s = _sacc(inp, cl=True) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_rho_tau_byte_equal(tmp_path): + """xi + rho/tau product: blocked tau covariance + verbatim RHO_STATS match.""" + inp = _inputs(seed=20) + reference = _reference_fits(tmp_path, inp, rho_tau=True) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + assert out.read_bytes() == reference.read_bytes() + + +# ============================================================================= +# Structural + teeth checks +# ============================================================================= + + +def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): + """The tau covariance block couples tau_0 and tau_2 (not block-diagonal). + + covdat_to_fits truncates the 3-statistic CosmoCov tau covariance to its + first 2 blocks and lays it in as ONE contiguous block, so tau_0<->tau_2 + cross-terms survive. A block-diagonal shortcut would zero them; pin that + the converter keeps them. + """ + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(out) as hdul: + cov = hdul["COVMAT"].data + # tau_0 block rows 2*N_ANG..3*N_ANG, tau_2 block 3*N_ANG..4*N_ANG. + cross = cov[2 * N_ANG : 3 * N_ANG, 3 * N_ANG : 4 * N_ANG] + expected = inp["tau_cov_full"][:N_ANG, N_ANG : 2 * N_ANG] + assert np.array_equal(cross, expected) + assert np.any(cross != 0.0) + + +def test_perturbed_xi_changes_output(tmp_path): + """Teeth: a changed xi+ input must change the converted data vector.""" + inp = _inputs(seed=0) + s = _sacc(inp) + out = tmp_path / "base.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + with fits.open(out) as hdul: + base_xip = hdul["XI_PLUS"].data["VALUE"].copy() + + inp2 = _inputs(seed=0) + inp2["xip"] = inp2["xip"] + 1.0 + s2 = _sacc(inp2) + out2 = tmp_path / "perturbed.fits" + twopoint_convert.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + with fits.open(out2) as hdul: + new_xip = hdul["XI_PLUS"].data["VALUE"] + + assert not np.array_equal(base_xip, new_xip) + assert np.array_equal(new_xip, inp2["xip"]) + + +def test_rho_tau_sidecars_required_together(tmp_path): + """Supplying only one of the rho/tau sidecars is a loud error.""" + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) + with pytest.raises(ValueError, match="together"): + twopoint_convert.sacc_to_twopoint_fits( + s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 + ) diff --git a/src/sp_validation/tests/test_twopoint_convert_realdata.py b/src/sp_validation/tests/test_twopoint_convert_realdata.py new file mode 100644 index 00000000..9d2739a6 --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert_realdata.py @@ -0,0 +1,268 @@ +"""Candide-local byte-compare of the converter against real 2pt-FITS products. + +Skipped unless a real product exists on disk (candide only; never committed). +It closes the loop end to end on real data: take a real CosmoSIS 2pt-FITS, build +an analysis SACC from its own contents, convert that SACC back to a 2pt-FITS, and +byte-compare. + +The reference is *not* the on-disk file directly. The committed on-disk products +were written by an older ``cosmosis_fitting.py`` (they carry a CELL_BB HDU and +order COVMAT before NZ_SOURCE); the converter reproduces the *current* script, +which the PR-4 migration will use to regenerate them. So the meaningful contract +-- "the converter equals the current writer" -- is tested by running the current +``cosmosis_fitting.py`` builders on the same real contents and byte-comparing the +converter against that. For transparency the test also records the direct diff +against the stale on-disk file, and asserts only that it differs *by whole HDUs* +(the extra CELL_BB), not in any shared block -- i.e. the drift is purely the +known HDU-set change, with no silent data corruption. + +Observed on 2026-07-10 for ``SP_v1.4.6_leak_corr`` and a ``glass_mock`` sibling: +converter == current-script byte for byte; converter vs on-disk differs only by +the CELL_BB HDU. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") +_REAL_FILES = { + "SP_v1.4.6_leak_corr": _DATA + / "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" + / "cosmosis_SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits", + "glass_mock_00001": _DATA / "glass_mock_00001" / "cosmosis_glass_mock_00001.fits", +} + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sacc_from_2pt_fits(hdul): + """Build an analysis SACC from a real CosmoSIS 2pt-FITS's own contents. + + Reads the NZ, ξ±, pseudo-Cℓ (EE/BB) and blocked covariance back out of the + product and lays them into the standard SACC layout — the inverse direction + the converter then undoes. τ± minus and Cℓ EB are not in the 2pt-FITS, so + they are stored as zeros with identity covariance sub-blocks (the converter + consumes only the ``+``/EE parts). + """ + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + n_ell = len(ell) + + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + xi_cov = covmat[: 2 * n, : 2 * n] + tau_joint = covmat[2 * n : 4 * n, 2 * n : 4 * n] + + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi(s, (0, 0), theta, xip, xim, grid="coarse") + sacc_io.add_pseudo_cl( + s, + (0, 0), + ell, + cl_ee, + cl_bb, + np.zeros(n_ell), + window_ells=np.arange(2, 102), + window_weights=np.ones((100, n_ell)), + ) + sacc_io.add_tau(s, (0, 0), 0, theta, tau0p, np.zeros(n)) + sacc_io.add_tau(s, (0, 0), 2, theta, tau2p, np.zeros(n)) + + source, psf = sacc_io.source_name(0), sacc_io.PSF_TRACER + idx = { + "xi": np.concatenate( + [ + s.indices(sacc_io.XI_PLUS, (source, source)), + s.indices(sacc_io.XI_MINUS, (source, source)), + ] + ), + "ee": s.indices(sacc_io.CL_EE, (source, source)), + "bb": s.indices(sacc_io.CL_BB, (source, source)), + "eb": s.indices(sacc_io.CL_EB, (source, source)), + "t0p": s.indices(sacc_io.TAU_PLUS.format(k=0), (source, psf)), + "t0m": s.indices(sacc_io.TAU_MINUS.format(k=0), (source, psf)), + "t2p": s.indices(sacc_io.TAU_PLUS.format(k=2), (source, psf)), + "t2m": s.indices(sacc_io.TAU_MINUS.format(k=2), (source, psf)), + } + full = np.zeros((len(s.mean), len(s.mean))) + full[np.ix_(idx["xi"], idx["xi"])] = xi_cov + full[np.ix_(idx["ee"], idx["ee"])] = covmat_cell + tau_pp = np.concatenate([idx["t0p"], idx["t2p"]]) + full[np.ix_(tau_pp, tau_pp)] = tau_joint + for key in ("bb", "eb"): + full[np.ix_(idx[key], idx[key])] = np.eye(n_ell) + for key in ("t0m", "t2m"): + full[np.ix_(idx[key], idx[key])] = np.eye(n) + s.add_covariance(full) + + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + return s, hdul["RHO_STATS"].copy(), tau_sidecar + + +def _current_script_reference(cf, hdul, tmp_path): + """Reference: run the current cosmosis_fitting.py on the real file's contents.""" + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + np.savetxt(tmp_path / "nz.txt", np.column_stack([z, nz])) + np.savetxt(tmp_path / "cov.txt", covmat[: 2 * n, : 2 * n]) + tau_cov = np.zeros((3 * n, 3 * n)) + tau_cov[: 2 * n, : 2 * n] = covmat[2 * n : 4 * n, 2 * n : 4 * n] + np.save(tmp_path / "cov_tau.npy", tau_cov) + cl_block = np.zeros((5, len(ell))) + cl_block[0], cl_block[1], cl_block[4] = ell, cl_ee, cl_bb + np.save(tmp_path / "cl.npy", cl_block) + np.save(tmp_path / "cl_cov.npy", covmat_cell) + + nz_hdu = cf.nz_to_fits(str(tmp_path / "nz.txt")) + xip_hdu = cf._create_2pt_hdu(xip, theta, "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(xim, theta, "XI_MINUS", "G-R", "G-R") + cov_hdu = cf.covdat_to_fits( + str(tmp_path / "cov.txt"), filename_cov_tau=str(tmp_path / "cov_tau.npy") + ) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(tmp_path / "cl.npy")) + cl_ee_hdu, _ = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(tmp_path / "cl_cov.npy"), cov_hdu="COVAR_FULL") + + fits.HDUList([fits.PrimaryHDU(), hdul["RHO_STATS"].copy()]).writeto( + tmp_path / "rho.fits", overwrite=True + ) + rho_hdu = cf.rho_to_fits(str(tmp_path / "rho.fits"), theta=theta) + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + fits.HDUList([fits.PrimaryHDU(), tau_sidecar]).writeto( + tmp_path / "tau.fits", overwrite=True + ) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tmp_path / "tau.fits"), theta=theta) + + out = tmp_path / "reference.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + nz_hdu, + cov_hdu, + cov_cl_hdu, + xip_hdu, + xim_hdu, + cl_ee_hdu, + tau0_hdu, + tau2_hdu, + rho_hdu, + ] + ).writeto(out, overwrite=True) + return out + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_roundtrip_byte_equal(label, tmp_path): + """Converter reproduces the current writer byte for byte on a real product.""" + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + cf = _load_cf() + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + reference = _current_script_reference(cf, hdul, tmp_path) + + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + # The contract: converter == current cosmosis_fitting.py, byte for byte. + assert converted.read_bytes() == reference.read_bytes() + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): + """The stale on-disk file differs from the converter *only* by the CELL_BB HDU. + + Documents (and guards) the known script-version drift: the on-disk products + were written before CELL_BB was dropped from the assembly, so they carry one + extra HDU. Every HDU the two share carries the same data to floating-point + precision — the drift is a whole-HDU addition, never a silent change to a + shared block. (Bin-edge columns differ by ~1e-17 float noise: the stale file + stored a clean ``Z_LOW=0``, while ``z_mid - step/2`` rounds to ``-1.7e-18``; + both are the same number, so the shared-data check is ``allclose``, not + bitwise — bitwise equality is asserted against the *current* writer above.) + """ + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + ondisk_names = [h.name for h in hdul] + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(converted) as conv: + conv_names = [h.name for h in conv] + # The only HDU the on-disk file has that the converter does not. + assert set(ondisk_names) - set(conv_names) == {"CELL_BB"} + # Every shared table HDU carries the same data to float precision. + for name in conv_names: + if name == "PRIMARY": + continue + a, b = hdul[name].data, conv[name].data + if hasattr(a, "names"): + assert all(np.allclose(a[c], b[c]) for c in a.names), name + else: + assert np.allclose(a, b), name diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py new file mode 100644 index 00000000..c06e0eb3 --- /dev/null +++ b/src/sp_validation/twopoint_convert.py @@ -0,0 +1,330 @@ +"""TWOPOINT_CONVERT. + +:Name: twopoint_convert.py + +:Description: Convert an analysis SACC file into the "2pt FITS" that CosmoSIS's + ``2pt_like`` (and Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) + reads. The output reproduces today's hand-assembled product from + ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and + byte-for-byte: an NZDATA table, XI_PLUS / XI_MINUS 2pt tables, + optional CELL_EE / CELL_BB pseudo-Cℓ tables, the blocked COVMAT + (with ``STRT_i`` block-offset headers) and separate COVMAT_CELL, + and — when the SACC carries them — the TAU_{0,2}_PLUS 2pt tables + and the RHO_STATS table. + + The converter is the *inverse* of the SACC writers in + :mod:`sp_validation.sacc_io`: it reads statistics back through + those readers and lays them into the DES ``twopoint`` FITS + convention. SACC's canonical order is pair-major (per pair + ``[ξ+; ξ−]``); the 2pt-FITS layout is type-major (all ξ+, then all + ξ−), so the data-vector and its covariance are permuted here via + ``s.indices`` rather than assuming any global order. + + Scope note (single-bin today, tomography-ready): the assembly this + mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ + ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly. + A tomographic 2pt-FITS layout (multiple bin pairs, per-pair + BIN1/BIN2, one NZ column per bin) is a later extension; it is not + what today's CosmoSIS pipeline consumes, so it is out of scope for + the byte-compatible converter. + + Rho/tau caveat: the SACC layout stores ρ±/τ± *values* only, while + the 2pt-FITS RHO_STATS table also carries the per-mode *variances* + (``varrho_*``) that Sacha's fork's covariance path reads. Those + variances are not recoverable from the analysis SACC. The + converter therefore writes RHO_STATS / TAU HDUs only when a + ``rho_stats``/``tau_stats`` sidecar FITS is supplied (the same + file today's assembly copies verbatim); it never fabricates + variances. ξ±, Cℓ, n(z) and the covariance — the data vector + CosmoSIS fits — are fully reconstructed from SACC alone. +""" + +import numpy as np +from astropy.io import fits + +from . import sacc_io + +# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the +# extension name — copied from cosmosis_fitting.py so the headers match card +# for card. +_QUANT = { + "XI_PLUS": ("G+R", "G+R"), + "XI_MINUS": ("G-R", "G-R"), + "CELL_EE": ("GEF", "GEF"), + "CELL_BB": ("GBF", "GBF"), + "TAU_0_PLUS": ("G+R", "P+R"), + "TAU_2_PLUS": ("G+R", "SR+R"), +} + + +def _twopoint_hdu(name, values, ang, *, ang_unit=None): + """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). + + Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: + same column order and formats, the ``2PTDATA`` marker, the QUANT pair for + ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG + column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). + """ + nbins = len(values) + angbin = np.arange(1, nbins + 1) + columns = [ + fits.Column(name="BIN1", format="K", array=np.ones(nbins)), + fits.Column(name="BIN2", format="K", array=np.ones(nbins)), + fits.Column(name="ANGBIN", format="K", array=angbin), + fits.Column(name="VALUE", format="D", array=values), + fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), + ] + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) + quant1, quant2 = _QUANT[name] + for key, value in { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + }.items(): + hdu.header[key] = value + return hdu + + +def _nz_hdu(s, n_bins): + """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. + + Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` + grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` + column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins + are required to share the ``z`` grid — the single ``Z_MID`` axis of the + DES NZDATA table. + """ + z_mid, nz0 = sacc_io.get_nz(s, 0) + z_mid = np.asarray(z_mid, dtype=float) + step = z_mid[1] - z_mid[0] + z_low = z_mid - step / 2 + z_high = z_mid + step / 2 + + columns = [ + fits.Column(name="Z_LOW", format="D", array=z_low), + fits.Column(name="Z_MID", format="D", array=z_mid), + fits.Column(name="Z_HIGH", format="D", array=z_high), + ] + for i in range(n_bins): + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the DES " + "NZDATA table requires one shared Z_MID axis" + ) + columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) + + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") + for key, value in { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": n_bins, + "NZ": len(z_low), + }.items(): + hdu.header[key] = value + return hdu + + +def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): + """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. + + Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for + card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed + and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` + CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME + card is created early, with astropy's standard comment) before re-stamping. + ``name_in_ctor`` selects the second form so the card order matches exactly. + """ + matrix = np.asarray(matrix, dtype=np.float64) + if matrix.shape[0] != matrix.shape[1]: + raise ValueError(f"covariance must be square; got shape {matrix.shape}") + hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) + hdu.header["COVDATA"] = "True" + hdu.header["EXTNAME"] = extname + for i, (name, start) in enumerate(zip(block_names, block_starts)): + hdu.header[f"NAME_{i}"] = name + hdu.header[f"STRT_{i}"] = int(start) + return hdu + + +def _type_major_xi(s, bins): + """Return ``(theta, xip, xim)`` for one bin pair from the SACC coarse grid. + + ``sacc_io.get_xi`` already returns each statistic in insertion (= ascending + θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two + arrays it hands back, so no further permutation is needed for a single pair. + """ + return sacc_io.get_xi(s, bins, grid="coarse") + + +def sacc_to_twopoint_fits( + s, + path, + *, + rho_stats_hdu=None, + tau_stats_hdu=None, + n_bins=1, +): + """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. + + The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product + for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then + (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, + and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, + RHO_STATS. The data vector and its covariance are laid out type-major + (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` + convention CosmoSIS reads. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC (coarse ξ±, optional pseudo-Cℓ, covariance, and — for the + ρ/τ product — the τ data points; see ``rho_stats_hdu``). + path : str + Output FITS path (overwritten). + rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional + The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's + assembly does. Required together to write the ρ/τ product; the SACC + alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When + omitted, a pure ξ (± Cℓ) product is written. + n_bins : int, optional + Number of source tomographic bins (default 1, the current single-bin + analysis). Sets the NZDATA column count. + + Returns + ------- + astropy.io.fits.HDUList + The assembled list, also written to ``path``. + """ + if (rho_stats_hdu is None) != (tau_stats_hdu is None): + raise ValueError( + "rho_stats_hdu and tau_stats_hdu must be supplied together " + "(the ρ/τ product needs both, or neither for a pure-ξ product)" + ) + use_rho_tau = rho_stats_hdu is not None + bins = (0, 0) + + nz_hdu = _nz_hdu(s, n_bins) + theta, xip, xim = _type_major_xi(s, bins) + xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") + xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") + + cell_hdu, cov_cell_hdu = _build_cell(s, bins) + + cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) + + tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) + + # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, + # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. + hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] + if cov_cell_hdu is not None: + hdu_list.append(cov_cell_hdu) + hdu_list.extend([xip_hdu, xim_hdu]) + if cell_hdu is not None: + hdu_list.append(cell_hdu) + if use_rho_tau: + hdu_list.extend([*tau_hdus, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(path, overwrite=True) + return hdul + + +def _build_cell(s, bins): + """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. + + Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is + emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's + assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). + The SACC still carries EE/BB/EB with bandpower windows for the B-mode + null-test path; this converter surfaces only the block CosmoSIS reads. The + CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL + ImageHDU, matching today's product. + """ + if sacc_io.CL_EE not in s.get_data_types(): + return None, None + + ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) + cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) + cell_idx = s.indices(sacc_io.CL_EE, sacc_io._pair(bins)) + cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] + cov_cell_hdu = _cov_hdu( + cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True + ) + return cell_hdu, cov_cell_hdu + + +def _build_covmat(s, bins, *, use_rho_tau): + """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). + + The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for + the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under + ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally + with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. + """ + pair = sacc_io._pair(bins) + idx_p = s.indices(sacc_io.XI_PLUS, pair) + idx_m = s.indices(sacc_io.XI_MINUS, pair) + n_theta = len(idx_p) + xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation + xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + names = ["XI_PLUS", "XI_MINUS"] + starts = [0, n_theta] + matrix = xi_cov + + if use_rho_tau: + # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the + # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in + # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the + # SACC those two selections are not adjacent (τ_0− sits between them), so + # gather both index sets and extract the joint sub-block, ξ↔τ zero. + tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) + idx_tau0 = s.indices(sacc_io.TAU_PLUS.format(k=0), tau_pair) + idx_tau2 = s.indices(sacc_io.TAU_PLUS.format(k=2), tau_pair) + tau_idx = np.concatenate([idx_tau0, idx_tau2]) + tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] + matrix = _block_diag(matrix, tau_cov) + names += ["TAU_0_PLUS", "TAU_2_PLUS"] + starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] + + return _cov_hdu(matrix, names, starts) + + +def _block_diag(*blocks): + """Stack square blocks block-diagonally with zero cross-blocks.""" + sizes = [b.shape[0] for b in blocks] + n = sum(sizes) + out = np.zeros((n, n)) + start = 0 + for b in blocks: + out[start : start + b.shape[0], start : start + b.shape[0]] = b + start += b.shape[0] + return out + + +def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): + """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. + + Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` + columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied + verbatim from the sidecar with its θ column forced onto the ξ grid. The + ``varrho_*`` columns ride along in the copy — they are why the sidecar is + required (the SACC cannot supply them). + """ + if not use_rho_tau: + return (), None + + tau = tau_stats_hdu.data + tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") + tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") + + rho_hdu = rho_stats_hdu.copy() + rho_hdu.name = "RHO_STATS" + rho_hdu.data = rho_hdu.data.copy() + rho_hdu.data["theta"] = theta + return (tau0_hdu, tau2_hdu), rho_hdu From f9b1d12199c414246ee8b67bf9b079bec389ff00 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:27:10 +0200 Subject: [PATCH 07/38] feat(one_covariance): SACC <-> OneCovariance file-format glue Add sp_validation.one_covariance_io, the two small pieces of glue between the SACC layout and OneCovariance (which knows nothing of SACC and only exchanges files): - write_nz / nz_table / nz_config_stanza: the source_i NZ tracers of an analysis SACC -> OneCovariance's combined whitespace n(z) file (column 0 = shared z grid, one n(z) column per tomographic bin, no bin edges) plus the [redshift] config stanza pointing at it. Config keys verified against upstream OneCovariance config.ini (zlens_directory/zlens_file/value_loc_in_lensbin); the UNIONS template (cosmo_val/pseudo_cl.py._modify_onecov_config) uses the z_directory alias, exposed via dir_key. Fails fast on missing bins or disagreeing z grids. - covariance_blocks: OneCovariance's flat covariance_list_*.dat output -> dense square block(s) paired with SACC selectors for sacc_io.assemble_covariance, reusing statistics.cov_from_one_covariance for the per-block reshape (col 10 gaussian / col 9 gauss+non-gaussian). Single-block and multi-block (tomography-ready) forms. Tests (test_one_covariance_io.py, 9 passing): reshape to a hand-built matrix + gaussian/gauss+ng column selection + perturbation teeth; blocks feed assemble_covariance and round-trip s.covariance.dense; n(z) write/read round-trip, UNIONS dir_key, header/no-header, and fail-fast on mismatched grids / missing bin / bad value_loc. Co-Authored-By: Claude Fable 5 --- src/sp_validation/one_covariance_io.py | 256 ++++++++++++++++ .../tests/test_one_covariance_io.py | 274 ++++++++++++++++++ 2 files changed, 530 insertions(+) create mode 100644 src/sp_validation/one_covariance_io.py create mode 100644 src/sp_validation/tests/test_one_covariance_io.py diff --git a/src/sp_validation/one_covariance_io.py b/src/sp_validation/one_covariance_io.py new file mode 100644 index 00000000..4c0f2ae4 --- /dev/null +++ b/src/sp_validation/one_covariance_io.py @@ -0,0 +1,256 @@ +"""ONE_COVARIANCE_IO. + +:Name: one_covariance_io.py + +:Description: File-format glue between the SACC data-product layout + (:mod:`sp_validation.sacc_io`) and OneCovariance + (https://github.com/rreischke/OneCovariance). Two directions: + + - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the + ``source_i`` NZ tracers of an analysis SACC are written as the + combined whitespace-delimited redshift file OneCovariance reads + (column 0 = z grid, then one ``n(z)`` column per tomographic + bin, no bin edges), and a matching ``[redshift]`` config stanza + is returned via :func:`nz_config_stanza`. + + - **OneCovariance output -> SACC covariance blocks** + (:func:`covariance_blocks`): the flat ``covariance_list_*.dat`` + table OneCovariance emits (one row per element pair) is reshaped + into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the + per-block reshape — and paired with SACC selectors so a caller + can feed them straight to + :func:`sp_validation.sacc_io.assemble_covariance`. + + OneCovariance itself is *not* a dependency: this module only + touches its file formats, verified against the upstream + ``config.ini`` (``rreischke/OneCovariance`` @ main). + + n(z) file format (upstream ``config.ini`` comment, verbatim): + + ``redshift n_1(z) ... n_{N_source}(z)`` + + i.e. a plain whitespace-delimited text file, column 0 the shared + redshift grid and one column per tomographic bin — no ``z_low``/ + ``z_high`` edges (this is the OneCovariance convention, distinct + from the CosmoSIS NZDATA table which *does* carry edges). All + source bins must therefore share one z grid. + + ``[redshift]`` config keys (upstream canonical names): a single + combined file goes in ``zlens_directory`` + ``zlens_file``; + ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where + in each histogram bin the tabulated ``n(z)`` value sits — ``mid`` + for the bin-centred grids the SACC stores. NOTE: the UNIONS + OneCovariance template driven by + ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older + key names ``z_directory``/``zlens_file`` instead; pass + ``dir_key="z_directory"`` to match that template. +""" + +import os + +import numpy as np + +from . import sacc_io +from .statistics import cov_from_one_covariance + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = sacc_io.get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if sacc_io.source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.sacc_io.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``sacc_io.assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_one_covariance_io.py b/src/sp_validation/tests/test_one_covariance_io.py new file mode 100644 index 00000000..a99ad552 --- /dev/null +++ b/src/sp_validation/tests/test_one_covariance_io.py @@ -0,0 +1,274 @@ +"""Tests for :mod:`sp_validation.one_covariance_io`. + +All synthetic, all fast: the OneCovariance fixtures are built in memory +shaped exactly like its real file I/O — a flat ``covariance_list`` table with +the ``(i, j)`` index rows and the Gaussian / Gauss+non-Gaussian value columns +at the indices ``cov_from_one_covariance`` expects (col 10 / col 9), and the +combined n(z) text file (column 0 = z, one column per bin, no edges). No +cluster paths; OneCovariance is not imported. + +The two pieces: + +- **Piece 1** (n(z) SACC -> OneCovariance input): write the combined n(z) + file from a SACC's ``source_i`` NZ tracers, read it back, assert the z and + n(z) columns round-trip and the config stanza names the file. +- **Piece 2** (OneCovariance output -> SACC covariance blocks): reshape the + flat table to the dense block(s) and prove they feed + ``sacc_io.assemble_covariance`` cleanly (reshaped block -> assemble -> + ``s.covariance.dense`` matches the hand-built matrix). +""" + +import numpy as np +import numpy.testing as npt +import pytest + +from sp_validation import one_covariance_io as ocio +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic OneCovariance-shaped fixtures +# --------------------------------------------------------------------------- # +def _one_cov_table(cov_gauss, cov_all): + """Flatten two n x n matrices into a OneCovariance ``covariance_list`` table. + + Reproduces the real flat output: one row per ``(i, j)`` element pair in + row-major order ``k = i·n + j``, with the Gaussian value in column 10 and + the Gaussian+non-Gaussian value in column 9. Columns 0-8 and the index + columns are filled with self-documenting placeholder values (the reshape + only reads cols 9/10, but a realistic width proves it does not spill). + """ + n = cov_gauss.shape[0] + rows = [] + for i in range(n): + for j in range(n): + row = np.arange(11.0) # placeholder cols 0-8 (+ overwritten 9,10) + row[9] = cov_all[i, j] + row[10] = cov_gauss[i, j] + rows.append(row) + return np.array(rows) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n`` (a valid covariance).""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _nz(seed, n=40): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +# --------------------------------------------------------------------------- # +# Piece 2 — flat covariance_list -> dense block (reshape correctness + teeth) +# --------------------------------------------------------------------------- # +def test_covariance_blocks_reshapes_to_hand_built_matrix(): + """Pin the reshape and prove gaussian vs gauss+ng select different columns. + + WHAT IS PINNED: ``covariance_blocks`` flattens/reshapes the OneCovariance + ``covariance_list`` table into a dense square block matching a hand-built + covariance. It delegates the per-block reshape to + ``statistics.cov_from_one_covariance``, so column 10 (gaussian) and column 9 + (gauss+ng) must recover the two distinct hand-built matrices. + + WHY TEETH: (a) ``gaussian=True`` vs ``False`` must return the two *different* + matrices, proving the column flag is load-bearing; (b) perturbing a single + entry of the flat input must change exactly that entry of the reshaped + block, proving the reshape actually reads the table (not a constant). + """ + cov_gauss = _spd(4, seed=1) + cov_all = _spd(4, seed=2) + table = _one_cov_table(cov_gauss, cov_all) + + selector = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + + [(sel_g, block_g)] = ocio.covariance_blocks(table, selector, gaussian=True) + [(sel_a, block_a)] = ocio.covariance_blocks(table, selector, gaussian=False) + + assert sel_g == selector and sel_a == selector + npt.assert_allclose(block_g, cov_gauss, rtol=1e-12) + npt.assert_allclose(block_a, cov_all, rtol=1e-12) + + # TEETH: gaussian and gauss+ng select different columns -> different blocks. + assert not np.allclose(block_g, block_a) + + # TEETH: a perturbation of one flat-table entry moves exactly that block + # entry (row k = i·n + j, col 10 for gaussian). + perturbed = table.copy() + perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) + [(_, block_p)] = ocio.covariance_blocks(perturbed, selector, gaussian=True) + npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) + block_p[2, 1] = block_g[2, 1] + npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved + + +def test_covariance_blocks_multiblock_form(): + """Prove the tomography-ready multi-block form reshapes each sub-table. + + WHAT IS PINNED: passing ``selectors=None`` and a sequence of + ``(selector, sub_table)`` pairs reshapes each sub-table independently and + returns them paired with their selectors in order — the shape needed to map + a multi-statistic OneCovariance output onto several SACC selectors. + + WHY TEETH: the two sub-tables carry different matrices; if the function + reshaped only the first or mixed them, the second block would not match its + own hand-built matrix. + """ + cov_a, cov_b = _spd(3, seed=3), _spd(2, seed=4) + table_a = _one_cov_table(cov_a, _spd(3, seed=5)) + table_b = _one_cov_table(cov_b, _spd(2, seed=6)) + sel_a = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + sel_b = (sio.XI_MINUS, (sio.source_name(0), sio.source_name(0))) + + blocks = ocio.covariance_blocks( + [(sel_a, table_a), (sel_b, table_b)], None, gaussian=True + ) + + assert [s for s, _ in blocks] == [sel_a, sel_b] + npt.assert_allclose(blocks[0][1], cov_a, rtol=1e-12) + npt.assert_allclose(blocks[1][1], cov_b, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 2 — the reshaped block feeds assemble_covariance cleanly +# --------------------------------------------------------------------------- # +def test_covariance_blocks_feed_assemble_covariance(): + """Round-trip: reshaped block -> assemble_covariance -> dense matches. + + WHAT IS PINNED: the ``(selector, dense)`` pair ``covariance_blocks`` + returns is directly consumable by ``sacc_io.assemble_covariance``: assembled + onto a SACC whose only statistic is one ξ+ block, ``s.covariance.dense`` + must equal the hand-built OneCovariance matrix. This is the end-to-end + contract between the two modules. + + WHY TEETH: the block must tile the data vector exactly; if the reshape + produced the wrong size or the wrong selector, ``assemble_covariance`` would + raise (its contiguity/tiling/size checks), so a clean assemble + matching + dense is a real proof. + """ + theta = np.geomspace(1.0, 100.0, 4) + xip, xim = np.arange(4) * 1e-5, np.arange(4) * 2e-5 + s = sio.new_sacc({0: _nz(0)}) + sio.add_xi(s, (0, 0), theta, xip, xim, grid="coarse") + + # The ξ block spans ξ+ then ξ− for the pair -> 8 points, one contiguous + # block (pair-major, matching sacc_io's canonical order). + cov_gauss = _spd(8, seed=7) + table = _one_cov_table(cov_gauss, _spd(8, seed=8)) + pair = (sio.source_name(0), sio.source_name(0)) + selector = np.concatenate( + [s.indices(sio.XI_PLUS, pair), s.indices(sio.XI_MINUS, pair)] + ) + + blocks = ocio.covariance_blocks(table, selector, gaussian=True) + sio.assemble_covariance(s, blocks) + + npt.assert_allclose(s.covariance.dense, cov_gauss, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 1 — n(z) SACC -> OneCovariance input (round-trip + config stanza) +# --------------------------------------------------------------------------- # +def test_write_nz_roundtrips_and_names_file(tmp_path): + """Round-trip the n(z) file and check the config stanza names it. + + WHAT IS PINNED: ``write_nz`` writes the SACC ``source_i`` NZ tracers as the + OneCovariance combined file (column 0 = z, one column per bin, no z_low/ + z_high edges). ``read_nz`` recovers the z grid and every per-bin n(z) + column, and the returned ``[redshift]`` stanza names the exact file written + (directory + basename) plus ``value_loc_in_lensbin``. + + WHY TEETH: the z grid and each n(z) column must round-trip to the values the + SACC holds (drawn from a seeded RNG); a transposed write or a dropped column + would fail the per-bin comparison. The stanza's directory/file must match + the path actually written. + """ + z, nz0 = _nz(10) + _, nz1 = _nz(11) + s = sio.new_sacc({0: (z, nz0), 1: (z, nz1)}) + + path = tmp_path / "nz_onecov.txt" + stanza = ocio.write_nz(s, path, n_bins=2) + + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + assert nz_read.shape == (len(z), 2) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + npt.assert_allclose(nz_read[:, 1], nz1, rtol=1e-12) + + assert stanza["zlens_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz_onecov.txt" + assert stanza["value_loc_in_lensbin"] == "mid" + + +def test_write_nz_unions_template_dir_key(tmp_path): + """The UNIONS-template ``z_directory`` key is selectable via ``dir_key``. + + WHAT IS PINNED: the upstream OneCovariance key is ``zlens_directory``, but + the UNIONS template (pseudo_cl.py._modify_onecov_config) writes + ``z_directory``. ``dir_key="z_directory"`` produces that variant so the + stanza drops straight into the UNIONS template's ``[redshift]`` section. + """ + z, nz0 = _nz(12) + s = sio.new_sacc({0: (z, nz0)}) + stanza = ocio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") + assert "z_directory" in stanza and "zlens_directory" not in stanza + assert stanza["z_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz.txt" + + +def test_write_nz_fails_on_mismatched_z_grids(tmp_path): + """Fail fast when source bins do not share one redshift grid. + + WHAT IS PINNED: the OneCovariance combined file has a single redshift + column, so all bins must share the z grid. A bin on a different grid must + raise ``ValueError`` at write time, not silently mis-align. + """ + z0, nz0 = _nz(20) + z1_shifted, nz1 = _nz(21) + z1_shifted = z1_shifted + 0.1 # different grid + s = sio.new_sacc({0: (z0, nz0), 1: (z1_shifted, nz1)}) + with pytest.raises(ValueError, match="differs from source bin 0"): + ocio.write_nz(s, tmp_path / "bad.txt", n_bins=2) + + +def test_write_nz_no_header_roundtrips(tmp_path): + """The bare (header=False) file still round-trips numerically. + + WHAT IS PINNED: ``header=False`` writes a purely numeric file (no ``#`` + column-name line); ``read_nz`` recovers the same z grid and n(z) column, so + the header is cosmetic and never load-bearing for the numeric round-trip. + """ + z, nz0 = _nz(40) + s = sio.new_sacc({0: (z, nz0)}) + path = tmp_path / "bare.txt" + ocio.write_nz(s, path, n_bins=1, header=False) + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + + +def test_nz_config_stanza_rejects_bad_value_loc(): + """``value_loc_in_lensbin`` outside {mid,left,right} fails fast. + + WHAT IS PINNED: OneCovariance only accepts ``mid``/``left``/``right`` for + the histogram-bin value location; an invalid value is a config bug and must + raise ``ValueError`` rather than write a stanza OneCovariance will reject. + """ + with pytest.raises(ValueError, match="value_loc_in_lensbin"): + ocio.nz_config_stanza("/dir", "nz.txt", value_loc="center") + + +def test_write_nz_fails_on_missing_bin(tmp_path): + """Fail fast when a requested source bin is absent from the SACC. + + WHAT IS PINNED: requesting more bins than the SACC carries is a real config + bug; ``write_nz`` raises ``ValueError`` naming the missing tracer rather + than writing a short file. + """ + z, nz0 = _nz(30) + s = sio.new_sacc({0: (z, nz0)}) + with pytest.raises(ValueError, match="source_1"): + ocio.write_nz(s, tmp_path / "short.txt", n_bins=2) From c631418339ce603c40a9c391e730203b6ccc67b2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:29:06 +0200 Subject: [PATCH 08/38] fix(cosmo_val): tau covariance block derived from CovTauTh write-side MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The theoretical tau covariance is not (6·nbin)² over [τ+; τ−]. Read from the write-side (shear_psf_leakage.rho_tau_cov.CovTauTh.build_cov): it is a (3·nbin)² k-major matrix over {τ0, τ2, τ5} with plus/minus folded into one component per index — exactly the flavor today's CosmoSIS chain consumes via covdat_to_fits. rho_tau_to_sacc now scatters that plus-only block into the τ-plus rows/columns of the 6·nbin τ block (per-k [+;−] insertion order), leaves τ-minus a vartau diagonal, and keeps plus↔minus cross zero. The kwarg is renamed tau_cov -> tau_cov_th to name the flavor. tau_cov_th=None keeps a compact DiagonalCovariance placeholder. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/core.py | 18 +++++ src/sp_validation/cosmo_val/pseudo_cl.py | 33 +++++--- src/sp_validation/cosmo_val/real_space.py | 81 ++------------------ src/sp_validation/cosmo_val/sacc_writers.py | 60 ++++++++++----- src/sp_validation/tests/test_sacc_writers.py | 37 +++++---- 5 files changed, 108 insertions(+), 121 deletions(-) diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 48251bbb..0c9273d8 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -15,6 +15,7 @@ find_conservative_scale_cut_key, ) from ..statistics import chi2_and_pte +from ..version import __version__ from .catalog_characterization import CatalogCharacterizationMixin from .cosebis import CosebisMixin from .pseudo_cl import PseudoClMixin @@ -380,6 +381,23 @@ def _output_path(self, *parts): """ return os.path.abspath(os.path.join(self.cc["paths"]["output"], *parts)) + def sacc_nz(self, version): + """Single-bin ``nz`` mapping ``{0: (z, nz)}`` for the SACC writers. + + The tomography-native writer interface (``sacc_writers``) takes an nz + dict keyed by 0-based source bin; the round is single-bin, so the whole + survey n(z) is bin 0. ``get_redshift`` returns ``(z, nz)``. + """ + return {0: tuple(self.get_redshift(version))} + + def sacc_metadata(self, version): + """Provenance metadata stored on every SACC part for ``version``.""" + return { + "catalogue_version": version, + "sp_validation_version": __version__, + "npatch": self.npatch, + } + def get_redshift(self, version): """Load redshift distribution for a catalog version. diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 514049a6..0dee59f4 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -17,6 +17,7 @@ from astropy.io import fits from cs_util.cosmo import get_theo_c_ell +from .. import sacc_io from ..pseudo_cl import ( apply_random_rotation, get_n_gal_map, @@ -26,6 +27,10 @@ ) from ..rho_tau import get_params_rho_tau from ..statistics import chi2_and_pte, cov_from_one_covariance +from .sacc_writers import BIN as SACC_BIN + +# NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). +_NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 class PseudoClMixin: @@ -454,6 +459,12 @@ def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): def calculate_pseudo_cl(self): """ Compute the pseudo-Cl of given catalogs. + + Each version's spectra are born as a SACC part (``pseudo_cl_{ver}.sacc``) + via :func:`sacc_writers.pseudo_cl_to_sacc` — EE/BB/EB carrying the shared + NaMaster bandpower window. The in-memory ``self._pseudo_cls[ver]`` + ``"pseudo_cl"`` entry keeps the ``ELL``/``EE``/``EB``/``BB`` arrays the + plotting and B-mode-summary consumers read by column name. """ self.print_start("Computing pseudo-Cl's") @@ -468,11 +479,10 @@ def calculate_pseudo_cl(self): self._pseudo_cls[ver] = {} - out_path = self._output_path(f"pseudo_cl_{ver}.fits") + out_path = self._output_path(f"pseudo_cl_{ver}.sacc") if os.path.exists(out_path): self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) elif self.cell_method == "map": self.calculate_pseudo_cl_map(ver, nside, out_path) elif self.cell_method == "catalog": @@ -482,6 +492,13 @@ def calculate_pseudo_cl(self): self.print_done("Done pseudo-Cl's") + @staticmethod + def _load_pseudo_cl_sacc(out_path): + """Read a pseudo-Cl SACC part into the ELL/EE/EB/BB dict consumers use.""" + s = sacc_io.load(out_path) + ell, ee, bb, eb, _window = sacc_io.get_pseudo_cl(s, SACC_BIN) + return {"ELL": ell, "EE": ee, "EB": eb, "BB": bb} + def calculate_pseudo_cl_map(self, ver, nside, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -547,10 +564,9 @@ def calculate_pseudo_cl_map(self, ver, nside, out_path): cl_shear = cl_shear - cl_noise self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) def calculate_pseudo_cl_catalog(self, ver, out_path): params = get_params_rho_tau(self.cc[ver], survey=ver) @@ -563,10 +579,9 @@ def calculate_pseudo_cl_catalog(self, ver, out_path): ) self.print_cyan("Saving pseudo-Cl's...") - self.save_pseudo_cl(ell_eff, cl_shear, out_path) + self.pseudo_cl_to_sacc_part(ver, out_path, ell_eff, cl_shear, wsp) - cl_shear = fits.getdata(out_path) - self._pseudo_cls[ver]["pseudo_cl"] = cl_shear + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) def get_n_gal_map(self, params, nside, cat_gal): """Weighted galaxy number-density map (thin wrapper -> primitive).""" diff --git a/src/sp_validation/cosmo_val/real_space.py b/src/sp_validation/cosmo_val/real_space.py index 76d05d0a..a0852d77 100644 --- a/src/sp_validation/cosmo_val/real_space.py +++ b/src/sp_validation/cosmo_val/real_space.py @@ -12,12 +12,11 @@ import matplotlib.ticker as mticker import numpy as np import treecorr -from astropy.io import fits from cs_util import plots as cs_plots class RealSpaceMixin: - def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): + def calculate_2pcf(self, ver, npatch=None, **treecorr_config): """ Calculate the two-point correlation function (2PCF) ξ± for a given catalog version with TreeCorr. @@ -34,9 +33,6 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): npatch (int, optional): The number of patches to use for the calculation. Defaults to the instance's `npatch` attribute. - save_fits (bool, optional): Whether to save the ξ± results to FITS files. - Defaults to False. - **treecorr_config: Additional TreeCorr configuration parameters that will override the instance's default `treecorr_config`. For example, `min_sep=1`. @@ -49,8 +45,11 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): calculation is skipped, and the results are loaded from the file. - If a patch file for the given configuration does not exist, it is created during the process. - - FITS files for ξ+ and ξ− are saved with additional metadata in their - headers if `save_fits` is True. + - The ``.txt`` TreeCorr dump is the only raw byproduct written here + (read back by the covariance machinery and the skip-if-exists). The + analysis ξ± data product is born as SACC in the Snakemake scripts + (``run_2pcf.py`` coarse / ``run_2pcf_highres.py`` fine), which call + ``xi_to_sacc``; there is no DES-style ξ FITS writer anymore. """ self.print_magenta(f"Computing {ver} ξ±") @@ -101,74 +100,6 @@ def calculate_2pcf(self, ver, npatch=None, save_fits=False, **treecorr_config): gg.process(cat_gal) gg.write(out_fname, write_patch_results=True, write_cov=True) - # Save xi_p and xi_m results to fits file - # (moved outside so it runs even if txt exists) - if save_fits: - lst = np.arange(1, treecorr_config["nbins"] + 1) - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - - col1 = fits.Column(name="BIN1", format="K", array=np.ones(len(lst))) - col2 = fits.Column(name="BIN2", format="K", array=np.ones(len(lst))) - col3 = fits.Column(name="ANGBIN", format="K", array=lst) - col4 = fits.Column(name="VALUE", format="D", array=gg.xip) - col5 = fits.Column(name="ANG", format="D", unit="arcmin", array=gg.rnom) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - xiplus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_PLUS") - - col4 = fits.Column(name="VALUE", format="D", array=gg.xim) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5]) - ximinus_hdu = fits.BinTableHDU.from_columns(coldefs, name="XI_MINUS") - - # append xi_plus header info - xiplus_dict = { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - } - for key in xiplus_dict: - xiplus_hdu.header[key] = xiplus_dict[key] - # Use same naming format as txt output - fits_base = out_fname.replace(".txt", "").replace("_xi_", "_") - xiplus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_plus_{ver}')}.fits", - overwrite=True, - ) - - # append xi_minus header info - ximinus_dict = {**xiplus_dict, "QUANT1": "G-R", "QUANT2": "G-R"} - for key in ximinus_dict: - ximinus_hdu.header[key] = ximinus_dict[key] - ximinus_hdu.writeto( - f"{fits_base.replace(ver, f'xi_minus_{ver}')}.fits", - overwrite=True, - ) - # Add correlation object to class if not hasattr(self, "cat_ggs"): self.cat_ggs = {} diff --git a/src/sp_validation/cosmo_val/sacc_writers.py b/src/sp_validation/cosmo_val/sacc_writers.py index d227c379..76e9e6e0 100644 --- a/src/sp_validation/cosmo_val/sacc_writers.py +++ b/src/sp_validation/cosmo_val/sacc_writers.py @@ -124,15 +124,26 @@ def pure_eb_to_sacc(nz, metadata, theta, eb, covariance=None): return s -def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov=None): +def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov_th=None): """One ρ/τ part: ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage. ``rho_stats`` / ``tau_stats`` are the ``shear_psf_leakage`` handler tables (columns ``theta``, ``rho_{k}_p``, ``varrho_{k}_p``, ``rho_{k}_m``, … and - the τ analogue). ρ carries a diagonal (varxip/varxim) covariance — a - diagnostic placeholder, not used by inference — while τ carries ``tau_cov``, - the theoretical ``CovTauTh`` block the CosmoSIS τ-likelihood consumes. The - block order matches insertion: ρ (all +then−, per k) then τ. + the τ analogue). Both diagnostics stay out of the blind and only τ enters + inference, so the covariance is a block-diagonal placeholder except for the + τ-plus theory block: + + - ρ (all 6·nbin points): diagonal from ``varrho`` — a diagnostic placeholder, + not consumed by inference. + - τ (6·nbin points, per-k ``[τ+; τ−]``): the ``CovTauTh`` theory covariance + ``tau_cov_th`` scattered into the τ-plus rows/columns. ``CovTauTh.build_cov`` + returns a ``(3·nbin, 3·nbin)`` k-major matrix over ``{τ0, τ2, τ5}`` with the + plus/minus contributions folded into one component per k (verified against + the write-side); it therefore aligns to our τ-plus points ``{τ0+, τ2+, τ5+}`` + in k-major order, and today's CosmoSIS chain (``covdat_to_fits``) consumes + exactly this flavor for τ. The τ-minus points carry only a ``vartau`` + diagonal (no theory covariance for them exists). ``tau_cov_th=None`` falls + back to a fully diagonal τ block (a flagged placeholder, not the design). """ s = sio.new_sacc(nz, metadata) theta_rho = np.asarray(rho_stats["theta"]) @@ -154,6 +165,7 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov=None): np.asarray(tau_stats[f"tau_{k}_p"]), np.asarray(tau_stats[f"tau_{k}_m"]), ) + nbin = len(theta_tau) rho_var = np.concatenate( [ np.concatenate([rho_stats[f"varrho_{k}_p"], rho_stats[f"varrho_{k}_m"]]) @@ -166,22 +178,30 @@ def rho_tau_to_sacc(nz, metadata, rho_stats, tau_stats, tau_cov=None): for k in TAU_K ] ) - if tau_cov is None: - # Pure diagnostic file: diagonal covariance across ρ and τ. + if tau_cov_th is None: + # Fully diagonal placeholder — a DiagonalCovariance (compact, honest) for + # the standalone diagnostic file; assemble reads it back via .dense. s.add_covariance(np.concatenate([rho_var, tau_var])) - else: - # ρ diagonal (diagnostic) + τ dense theoretical block (inference input). - n_rho, n_tau = len(rho_var), len(tau_var) - tau_cov = np.asarray(tau_cov) - if tau_cov.shape != (n_tau, n_tau): - raise ValueError( - f"tau_cov shape {tau_cov.shape} does not match the {n_tau} τ " - "data points" - ) - full = np.zeros((n_rho + n_tau, n_rho + n_tau)) - full[:n_rho, :n_rho] = np.diag(rho_var) - full[n_rho:, n_rho:] = tau_cov - s.add_covariance(full) + return s + tau_cov_th = np.asarray(tau_cov_th) + n_plus = len(TAU_K) * nbin + if tau_cov_th.shape != (n_plus, n_plus): + raise ValueError( + f"tau_cov_th shape {tau_cov_th.shape} does not match the " + f"{n_plus} τ-plus points ({len(TAU_K)} indices × {nbin} bins) — " + "CovTauTh.build_cov returns one (plus-folded) component per τ index" + ) + n_rho, n_tau = len(rho_var), len(tau_var) + tau_block = np.diag(tau_var) + # τ-plus local positions in the τ block, k-major (per-k layout is [+; −]). + plus = np.concatenate( + [np.arange(2 * i * nbin, 2 * i * nbin + nbin) for i in range(len(TAU_K))] + ) + tau_block[np.ix_(plus, plus)] = tau_cov_th + full = np.zeros((n_rho + n_tau, n_rho + n_tau)) + full[:n_rho, :n_rho] = np.diag(rho_var) + full[n_rho:, n_rho:] = tau_block + s.add_covariance(full) return s diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index 057697e4..d886fa0a 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -192,32 +192,35 @@ def test_rho_tau_to_sacc_diagonal(tmp_path): def test_rho_tau_to_sacc_tau_theory_block(tmp_path): + """The (3·nbin) plus-only CovTauTh block scatters into the τ-plus rows/cols; + τ-minus keeps a vartau diagonal, and cross plus↔minus stays zero.""" rho, tau, theta = _rho_tau_tables() - n_tau = 2 * len(sw.TAU_K) * len(theta) - tau_cov = _spd(n_tau, 11) - s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov=tau_cov) + nbin = len(theta) + n_plus = len(sw.TAU_K) * nbin # τ-plus points (k-major, one component per k) + tau_cov_th = _spd(n_plus, 11) + s = sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=tau_cov_th) assert type(s.covariance).__name__ == "FullCovariance" - # τ sub-block equals the supplied theory covariance. tr = ("source_0", sio.PSF_TRACER) - tau_idx = np.concatenate( - [ - np.concatenate( - [ - s.indices(sio.TAU_PLUS.format(k=k), tr), - s.indices(sio.TAU_MINUS.format(k=k), tr), - ] - ) - for k in sw.TAU_K - ] + tau_plus = np.concatenate( + [s.indices(sio.TAU_PLUS.format(k=k), tr) for k in sw.TAU_K] + ) + tau_minus = np.concatenate( + [s.indices(sio.TAU_MINUS.format(k=k), tr) for k in sw.TAU_K] ) s2 = _roundtrip(s, tmp_path, "rttau") - assert np.allclose(s2.covariance.dense[np.ix_(tau_idx, tau_idx)], tau_cov) + dense = s2.covariance.dense + # τ-plus sub-block equals the supplied theory covariance (scatter is correct). + assert np.allclose(dense[np.ix_(tau_plus, tau_plus)], tau_cov_th) + # τ-minus is diagonal from vartau; plus↔minus cross is zero. + tau_minus_var = np.concatenate([np.asarray(tau[f"vartau_{k}_m"]) for k in sw.TAU_K]) + assert np.allclose(np.diag(dense[np.ix_(tau_minus, tau_minus)]), tau_minus_var) + assert np.allclose(dense[np.ix_(tau_plus, tau_minus)], 0.0) def test_rho_tau_to_sacc_tau_cov_shape_mismatch(): rho, tau, _ = _rho_tau_tables() - with pytest.raises(ValueError, match="tau_cov shape"): - sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov=_spd(3, 1)) + with pytest.raises(ValueError, match="tau_cov_th shape"): + sw.rho_tau_to_sacc({0: _nz()}, META, rho, tau, tau_cov_th=_spd(3, 1)) # --------------------------------------------------------------------------- # From 483961ffd3469b7cceecb27d48599e798cfb5afa Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:30:54 +0200 Subject: [PATCH 09/38] feat(cosmo_val): define pseudo_cl_to_sacc_part, complete pseudo-Cl SACC part MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit calculate_pseudo_cl_map/_catalog already call pseudo_cl_to_sacc_part (from the prior commit); this defines it — building the EE/BB/EB part via pseudo_cl_to_sacc(nz, meta, ell_eff, cl_all, wsp) and saving it as the native pseudo_cl_{ver}.sacc product. Drops the legacy save_pseudo_cl FITS writer (no external callers) and the unused _NMT_* constants. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/pseudo_cl.py | 36 ++++++++++-------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 0dee59f4..52d82849 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -28,9 +28,7 @@ from ..rho_tau import get_params_rho_tau from ..statistics import chi2_and_pte, cov_from_one_covariance from .sacc_writers import BIN as SACC_BIN - -# NaMaster spin-2 × spin-2 decoupled-spectrum row order (EE, EB, BE, BB). -_NMT_EE, _NMT_EB, _NMT_BB = 0, 1, 3 +from .sacc_writers import pseudo_cl_to_sacc class PseudoClMixin: @@ -670,26 +668,22 @@ def apply_random_rotation(self, e1, e2, rng=None): """ return apply_random_rotation(e1, e2, rng) - def save_pseudo_cl(self, ell_eff, pseudo_cl, out_path): - """ - Save pseudo-Cl's to a FITS file. + def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): + """Write the pseudo-Cl SACC part (EE/BB/EB + shared bandpower window). - Parameters - ---------- - pseudo_cl : np.array - Pseudo-Cl's to save. - out_path : str - Path to save the pseudo-Cl's to. + ``cl_all`` is NaMaster's decoupled ``(4, nbp)`` array (EE, EB, BE, BB); + the writer takes the shared bandpower window from ``wsp``. No covariance + is attached here — the analysis file's pseudo-Cl block is supplied at + assembly (``assemble_sacc``) from the NaMaster / OneCovariance product. """ - # Create columns of the fits file - col1 = fits.Column(name="ELL", format="D", array=ell_eff) - col2 = fits.Column(name="EE", format="D", array=pseudo_cl[0]) - col3 = fits.Column(name="EB", format="D", array=pseudo_cl[1]) - col4 = fits.Column(name="BB", format="D", array=pseudo_cl[3]) - coldefs = fits.ColDefs([col1, col2, col3, col4]) - cell_hdu = fits.BinTableHDU.from_columns(coldefs, name="PSEUDO_CELL") - - cell_hdu.writeto(out_path, overwrite=True) + s = pseudo_cl_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + ell_eff, + cl_all, + wsp, + ) + sacc_io.save(s, out_path) def plot_pseudo_cl(self): """ From 95e150fc23093c376c32b5e3c5836f275a027034 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:33:26 +0200 Subject: [PATCH 10/38] feat(cosmo_val): SACC part writers on cosebis / pure_eb / psf_systematics mixins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thin *_to_sacc_part methods that turn each already-computed diagnostic into its single-statistic SACC part via the born-as-SACC writers: - cosebis.py: cosebis_to_sacc_part picks the fiducial scale cut's {En,Bn,cov} (find_conservative_scale_cut_key, else the widest cut — mirroring plot_cosebis) and writes it; the multi-cut .npz sidecar (the PTE scan) is untouched. - pure_eb.py: pure_eb_to_sacc_part writes the six PURE_KEYS blocks at gg.meanr with the results['cov'] block. - psf_systematics.py: calculate_rho_tau_stats now writes a rho_tau_{base}.sacc part per version from the handler tables, passing cov_tau_{base}_th.npy as tau_cov_th when present (the tau-plus CovTauTh inference block), else a loud diagonal-placeholder fallback. nz/metadata come from the shared core helpers sacc_nz / sacc_metadata. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/cosebis.py | 42 +++++++++++++++++++ .../cosmo_val/psf_systematics.py | 36 ++++++++++++++++ src/sp_validation/cosmo_val/pure_eb.py | 21 ++++++++++ 3 files changed, 99 insertions(+) diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index aa4657ba..6b1d6146 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -7,6 +7,7 @@ import numpy as np +from .. import sacc_io from ..b_modes import ( calculate_cosebis, find_conservative_scale_cut_key, @@ -15,6 +16,7 @@ plot_cosebis_scale_cut_heatmap, save_cosebis_results, ) +from .sacc_writers import cosebis_to_sacc class CosebisMixin: @@ -140,6 +142,46 @@ def calculate_cosebis( return results + @staticmethod + def _fiducial_cosebis_result(results, fiducial_scale_cut): + """Select the fiducial scale cut's result dict + its ``(min, max)`` cut. + + ``calculate_cosebis`` returns either a single result dict (full range) or + a multi-cut mapping keyed by ``(theta_min, theta_max)`` tuples. Only the + fiducial cut is a SACC data product: pick it via + ``find_conservative_scale_cut_key`` when a fiducial cut is given, else the + widest cut — mirroring ``plot_cosebis``. + """ + multi_cut = isinstance(results, dict) and all( + isinstance(k, tuple) for k in results + ) + if not multi_cut: + return results, tuple(results["scale_cut"]) + key = ( + find_conservative_scale_cut_key(results, fiducial_scale_cut) + if fiducial_scale_cut is not None + else max(results, key=lambda x: x[1] - x[0]) + ) + return results[key], tuple(key) + + def cosebis_to_sacc_part(self, version, out_path, results, fiducial_scale_cut=None): + """Write the COSEBIs SACC part at the fiducial scale cut. + + ``results`` is the object ``calculate_cosebis`` returned (single dict or + multi-cut mapping). Only the fiducial cut's ``{En, Bn, cov}`` becomes the + part — a ``FullCovariance`` must cover every stored point and the cuts + overlap in mode space, so the non-fiducial cuts stay in the diagnostic + ``.npz`` sidecar. The nz/metadata are the version's. + """ + result, scale_cut = self._fiducial_cosebis_result(results, fiducial_scale_cut) + s = cosebis_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + result, + scale_cut, + ) + sacc_io.save(s, out_path) + def plot_cosebis( self, version=None, diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index 9f8a247e..af819977 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -16,10 +16,12 @@ from shear_psf_leakage.rho_tau_stat import PSFErrorFit from uncertainties import ufloat +from .. import sacc_io from ..rho_tau import ( get_rho_tau_w_cov, get_samples, ) +from .sacc_writers import rho_tau_to_sacc class PSFSystematicsMixin: @@ -41,11 +43,45 @@ def calculate_rho_tau_stats(self): cov_rho=self.compute_cov_rho, npatch=self.npatch, ) + self.rho_tau_to_sacc_part( + ver, out_dir, base, rho_stat_handler, tau_stat_handler + ) self.print_done("Rho stats finished") self._rho_stat_handler = rho_stat_handler self._tau_stat_handler = tau_stat_handler + def rho_tau_to_sacc_part( + self, version, out_dir, base, rho_stat_handler, tau_stat_handler + ): + """Write the ρ/τ SACC part for one version. + + ρ_0…ρ_5 autos and τ_0/τ_2/τ_5 leakage from the handler tables. The + ``CovTauTh`` theory covariance ``cov_tau_{base}_th.npy`` — a + ``(3·nbin, 3·nbin)`` plus-folded k-major block over ``{τ0, τ2, τ5}`` — is + passed as ``tau_cov_th`` when it exists (the τ-plus inference block); its + absence falls back to a diagonal placeholder for the whole part (loudly: + the τ inference block is then only a variance diagonal, not the theory + covariance). ρ always carries a diagnostic ``varrho`` diagonal. + """ + tau_cov_path = os.path.join(out_dir, f"cov_tau_{base}_th.npy") + tau_cov_th = np.load(tau_cov_path) if os.path.exists(tau_cov_path) else None + if tau_cov_th is None: + self.print_magenta( + f"No τ theory covariance at {tau_cov_path}; writing ρ/τ SACC part " + "with a diagonal placeholder covariance (τ inference block is a " + "variance diagonal, not CovTauTh)." + ) + s = rho_tau_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + rho_stat_handler.rho_stats, + tau_stat_handler.tau_stats, + tau_cov_th=tau_cov_th, + ) + out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") + sacc_io.save(s, out_path) + @property def rho_stat_handler(self): if not hasattr(self, "_rho_stat_handler"): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 7524074e..416ad8ef 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -7,6 +7,7 @@ import numpy as np +from .. import sacc_io from ..b_modes import ( calculate_eb_statistics, calculate_pure_eb_correlation, @@ -16,6 +17,7 @@ plot_pure_eb_correlations, save_pure_eb_results, ) +from .sacc_writers import pure_eb_to_sacc class PureEBMixin: @@ -132,6 +134,25 @@ def calculate_pure_eb( return results + def pure_eb_to_sacc_part(self, version, out_path, results): + """Write the pure-E/B SACC part (six ``PURE_KEYS`` blocks + covariance). + + ``results`` is the dict ``calculate_pure_eb`` returned: the six pure-mode + arrays under ``sacc_io.PURE_KEYS``, the ``"cov"`` block (in ``PURE_KEYS`` + order), and the reporting-grid TreeCorr object ``"gg"`` whose ``meanr`` + is the shared ``theta``. + """ + theta = results["gg"].meanr + eb = {key: results[key] for key in sacc_io.PURE_KEYS} + s = pure_eb_to_sacc( + self.sacc_nz(version), + self.sacc_metadata(version), + theta, + eb, + covariance=results["cov"], + ) + sacc_io.save(s, out_path) + def plot_pure_eb( self, versions=None, From ae03ea677c21d857c4666b7fd4ea0194621fd956 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 03:38:04 +0200 Subject: [PATCH 11/38] feat(workflow): born-as-SACC scripts (coarse/fine xi, pseudo-Cl, assemble) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run_2pcf.py: after calculate_2pcf, write {ver}_xi_coarse.sacc via xi_to_sacc(grid='coarse', theta_nom=rnom, npairs, weight) — no covariance (added at assembly). Drops the save_fits threading (mixin no longer writes DES-style xi FITS). - run_2pcf_highres.py: on rank 0, write the terminal {version}_xi_fine.sacc via xi_to_sacc(grid='fine', variances=[varxip; varxim]) as a DiagonalCovariance. Deletes write_xi_fits (DES-FITS); keeps the .txt. sacc_io import works on the bare-host MPI path (no healpy), n(z) read from shear.redshift_path. - generate_pseudo_cl.py: native product is now pseudo_cl_{ver}.sacc (the mixin writes it); report reads it back via sacc_io.get_pseudo_cl. - assemble_sacc.py (new, dual-mode): loads the per-statistic parts in canonical order and calls assemble_analysis_sacc. Injects the xi-coarse block from the CosmoCov .txt (already [xi+;xi-]-ordered) and the pseudo-Cl block from the NaMaster iNKA cov FITS as block-diagonal [EE_EE;BB_BB;EB_EB] (cross-spectrum blocks dropped, TODO flagged). --allow-placeholder attaches a documented diagonal for cov-less parts so DAG dry-runs / tests produce a valid FullCovariance; production requires the real cov inputs (fail-fast otherwise). Co-Authored-By: Claude Fable 5 --- workflow/scripts/assemble_sacc.py | 228 +++++++++++++++++++++++++ workflow/scripts/generate_pseudo_cl.py | 34 ++-- workflow/scripts/run_2pcf.py | 45 ++++- workflow/scripts/run_2pcf_highres.py | 74 +++++--- 4 files changed, 327 insertions(+), 54 deletions(-) create mode 100644 workflow/scripts/assemble_sacc.py diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py new file mode 100644 index 00000000..2ad6fa27 --- /dev/null +++ b/workflow/scripts/assemble_sacc.py @@ -0,0 +1,228 @@ +"""Assemble the terminal ``{version}.sacc`` analysis file from per-statistic parts. + +Dual-mode. Under Snakemake (``script:`` directive) the injected ``snakemake`` +object supplies the parts + covariance inputs; as a standalone CLI (argparse) +the same assembly runs from explicit flags (the lightcone/ASTRA path). + +Each per-statistic ``*.sacc`` *part* (written born-as-SACC by the mixins and the +run_2pcf / generate_pseudo_cl scripts) holds one statistic. The assembler loads +them in canonical order — ξ± coarse, pseudo-Cℓ, COSEBIs, pure-E/B, ρ/τ — and +calls :func:`sacc_writers.assemble_analysis_sacc`, which rebuilds one Sacc with a +single block-diagonal ``FullCovariance`` (point-insertion order = block order, +validated by ``sacc_io.assemble_covariance``). + +Covariance sourcing (the part-by-part decision) +----------------------------------------------- +``assemble_analysis_sacc`` REQUIRES every part to carry its own covariance block. +The COSEBIs, pure-E/B and ρ/τ parts already do (their writers attach it). The +ξ± coarse and pseudo-Cℓ parts are born cov-less by design; this script injects +their blocks before assembly: + +* **ξ± coarse** — the CosmoCov theory covariance ``.txt`` (``--xi-cov``). For the + single-bin round it is already ``[ξ+; ξ−]``-ordered (CosmoCov / covdat_to_fits: + ``STRT_0=0`` XI_PLUS, ``STRT_1=len/2`` XI_MINUS), which is exactly the SACC + ξ insertion order, so ``np.loadtxt`` → ``add_covariance`` needs no permutation. +* **pseudo-Cℓ** — the NaMaster iNKA / OneCovariance covariance FITS + (``--pseudo-cl-cov`` + ``--pseudo-cl-cov-hdu``). The FITS carries the 16 + EE/EB/BE/BB cross-blocks (each ``nbp × nbp``); SACC stores EE, BB, EB (in that + order), so we assemble the block-diagonal ``[EE_EE; BB_BB; EB_EB]``. The + cross-spectrum blocks (EE↔BB, …) are dropped — matching how the B-mode PTE + today reads only ``COVAR_BB_BB``. **TODO(PR-cov):** carry the full dense + EE/BB/EB cross-covariance once the analysis needs cross-spectrum correlations. + +When a cov input is absent the assembly cannot proceed on a real product; pass +``--allow-placeholder`` to attach a documented diagonal placeholder +(``placeholder_var`` on every point of the cov-less parts) so the DAG dry-run and +the fast test can still produce a structurally-valid ``FullCovariance``. The +placeholder is a flagged stand-in, never a science covariance. +""" + +import argparse + +import numpy as np + +from sp_validation import sacc_io +from sp_validation.cosmo_val.sacc_writers import assemble_analysis_sacc + +# NaMaster iNKA covariance FITS: per-spectrum HDU names. SACC insertion order is +# EE, BB, EB, so the block-diagonal is assembled in that order. +_CL_HDU = {"EE": "COVAR_EE_EE", "BB": "COVAR_BB_BB", "EB": "COVAR_EB_EB"} +_CL_ORDER = ("EE", "BB", "EB") + +# Canonical part order — the order assemble_analysis_sacc inserts points in, which +# must match the covariance block order. Missing parts are simply skipped. +CANONICAL = ("xi_coarse", "pseudo_cl", "cosebis", "pure_eb", "rho_tau") + + +def _pseudo_cl_cov_block(cov_fits, hdu): + """Block-diagonal ``[EE_EE; BB_BB; EB_EB]`` from the NaMaster iNKA cov FITS. + + ``hdu`` selects the file flavor: for the per-spectrum iNKA file we read the + three named diagonal HDUs; for a single dense HDU (OneCovariance / g+ng + ``COVAR_FULL``) that already spans EE/BB/EB we return it as-is. + """ + from astropy.io import fits + + with fits.open(cov_fits) as hdul: + names = {h.name for h in hdul} + if all(_CL_HDU[s] in names for s in _CL_ORDER): + blocks = [np.asarray(hdul[_CL_HDU[s]].data, float) for s in _CL_ORDER] + n = blocks[0].shape[0] + full = np.zeros((3 * n, 3 * n)) + for i, block in enumerate(blocks): + full[i * n : (i + 1) * n, i * n : (i + 1) * n] = block + return full + return np.asarray(hdul[hdu].data, float) + + +def _attach_cov(part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var): + """Ensure ``part`` carries a covariance, injecting the xi/pseudo-Cℓ block. + + ``part`` is mutated in place. cosebis/pure_eb/rho_tau parts already carry + their covariance and pass straight through. Raises loudly if a required xi / + pseudo-Cℓ block is missing and no placeholder was requested. + """ + if part.covariance is not None: + return part + if name == "xi_coarse": + if xi_cov is not None: + part.add_covariance(np.loadtxt(xi_cov)) + return part + elif name == "pseudo_cl": + if pseudo_cl_cov is not None: + part.add_covariance(_pseudo_cl_cov_block(pseudo_cl_cov, pseudo_cl_cov_hdu)) + return part + if placeholder_var is None: + raise ValueError( + f"the {name!r} part carries no covariance and no covariance input was " + f"given (--xi-cov / --pseudo-cl-cov). Supply the block, or pass " + "--allow-placeholder to attach a documented diagonal placeholder." + ) + part.add_covariance(np.full(len(part.mean), float(placeholder_var))) + return part + + +def assemble_sacc( + version, + part_paths, + out_path, + *, + xi_cov=None, + pseudo_cl_cov=None, + pseudo_cl_cov_hdu="COVAR_FULL", + placeholder_var=None, +): + """Assemble ``{version}.sacc`` from the per-statistic ``part_paths`` mapping. + + Parameters + ---------- + version : str + Catalogue version (stored in the assembled file's metadata). + part_paths : dict + ``{statistic: path}`` with statistic in :data:`CANONICAL`. Only the + present statistics are assembled; order is forced to canonical. + out_path : str + Destination ``{version}.sacc``. + xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var + Covariance sourcing — see the module docstring. + """ + parts = [] + nz = metadata = None + for name in CANONICAL: + path = part_paths.get(name) + if path is None: + continue + part = sacc_io.load(path) + if nz is None: + # The nz tracers + metadata are identical across parts (same version); + # take them from the first loaded part for the assembled file. + nz = {i: sacc_io.get_nz(part, i) for i in range(_n_source_bins(part))} + metadata = dict(part.metadata) + parts.append( + _attach_cov( + part, name, xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var + ) + ) + if not parts: + raise ValueError(f"no parts found for {version}: {part_paths}") + s = assemble_analysis_sacc(nz, metadata, parts) + sacc_io.save(s, out_path) + print(f"Assembled {len(parts)} parts -> {out_path}") + return s + + +def _n_source_bins(part): + """Count the ``source_{i}`` NZ tracers on a part (single-bin round -> 1).""" + i = 0 + while sacc_io.source_name(i) in part.tracers: + i += 1 + return i + + +def _from_snakemake(smk): + p = smk.params + inp = smk.input + part_paths = { + name: getattr(inp, name) + for name in CANONICAL + if hasattr(inp, name) and getattr(inp, name) + } + assemble_sacc( + version=p["version"], + part_paths=part_paths, + out_path=str(smk.output[0]), + xi_cov=getattr(inp, "xi_cov", None), + pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), + pseudo_cl_cov_hdu=p.get("pseudo_cl_cov_hdu", "COVAR_FULL"), + placeholder_var=p.get("placeholder_var", None), + ) + + +def _from_cli(argv=None): + ap = argparse.ArgumentParser( + description="Assemble the terminal {version}.sacc from per-statistic parts." + ) + ap.add_argument("--version", required=True, help="Catalogue version") + ap.add_argument("--out", required=True, help="Output {version}.sacc path") + for name in CANONICAL: + ap.add_argument( + f"--{name.replace('_', '-')}", default=None, help=f"{name} part" + ) + ap.add_argument("--xi-cov", default=None, help="CosmoCov ξ covariance .txt") + ap.add_argument( + "--pseudo-cl-cov", + default=None, + help="NaMaster/OneCovariance pseudo-Cℓ cov FITS", + ) + ap.add_argument( + "--pseudo-cl-cov-hdu", + default="COVAR_FULL", + help="HDU name for a single dense pseudo-Cℓ cov (EE/BB/EB-spanning)", + ) + ap.add_argument( + "--allow-placeholder", + type=float, + default=None, + metavar="VAR", + help="Attach a diagonal placeholder (variance VAR) to cov-less parts", + ) + a = ap.parse_args(argv) + part_paths = {name: getattr(a, name) for name in CANONICAL if getattr(a, name)} + assemble_sacc( + version=a.version, + part_paths=part_paths, + out_path=a.out, + xi_cov=a.xi_cov, + pseudo_cl_cov=a.pseudo_cl_cov, + pseudo_cl_cov_hdu=a.pseudo_cl_cov_hdu, + placeholder_var=a.allow_placeholder, + ) + + +if __name__ == "__main__": + try: + snakemake # noqa: F821 — injected by Snakemake's script: directive + except NameError: + _from_cli() + else: + _from_snakemake(snakemake) # noqa: F821 diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index a5c19a56..81f69414 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -4,12 +4,13 @@ object supplies the parameters and the native product is renamed to the tagged output filename the rule declares; as a standalone CLI (argparse) the same compute runs from explicit flags and the primitive's native -``pseudo_cl_{ver}.fits`` is left in place under ``--out`` (no rename — each +``pseudo_cl_{ver}.sacc`` is left in place under ``--out`` (no rename — each lc/ASTRA recipe gets its own output directory, so the untagged native name is unambiguous and the primitives' skip-if-exists never collides across nbins -runs). The CLI form is what the lightcone/ASTRA recipe calls, so the -measurement is driven directly (no nested Snakemake) with lc handling -orchestration: +runs). The C_ell data vector is born as SACC (EE/BB/EB with a shared bandpower +window) — see ``sp_validation.cosmo_val.sacc_writers.pseudo_cl_to_sacc``. The +CLI form is what the lightcone/ASTRA recipe calls, so the measurement is driven +directly (no nested Snakemake) with lc handling orchestration: python generate_pseudo_cl.py \ --ver SP_v1.4.6.3_leak_corr \ @@ -28,8 +29,7 @@ import json import os -from astropy.io import fits - +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation @@ -52,8 +52,8 @@ def generate_pseudo_cl( version : str Catalog version (e.g., "SP_v1.4.6_leak_corr") output_dir : str - Directory the pseudo-Cl FITS file is written into. The primitive writes - its native ``pseudo_cl_{version}.fits`` here; callers that need a tagged + Directory the pseudo-Cl SACC part is written into. The primitive writes + its native ``pseudo_cl_{version}.sacc`` here; callers that need a tagged filename rename it themselves (see ``_from_snakemake``). cat_config : str Path to catalog configuration YAML @@ -76,7 +76,7 @@ def generate_pseudo_cl( Returns ------- str - Path to the primitive's native ``pseudo_cl_{version}.fits`` product. + Path to the primitive's native ``pseudo_cl_{version}.sacc`` product. """ os.makedirs(output_dir, exist_ok=True) @@ -135,17 +135,17 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) - # Calculate pseudo-Cls only (no covariance) + # Calculate pseudo-Cls only (no covariance). The data vector is born as a + # SACC part: pseudo_cl_{version}.sacc under output_dir. cv.calculate_pseudo_cl() - # Report on the native product (renamed by the Snakemake caller, if any) - src_cl = os.path.join(output_dir, f"pseudo_cl_{version}.fits") + # Report on the native product (renamed by the Snakemake caller, if any). + src_cl = os.path.join(output_dir, f"pseudo_cl_{version}.sacc") if os.path.exists(src_cl): - with fits.open(src_cl) as hdul: - data = hdul["PSEUDO_CELL"].data - n_ell = len(data["ELL"]) - print(f"Generated pseudo-Cl with {n_ell} ell bins") - print(f"ell range: [{data['ELL'].min():.1f}, {data['ELL'].max():.1f}]") + s = sacc_io.load(src_cl) + ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] + print(f"Generated pseudo-Cl with {len(ell)} ell bins") + print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") return src_cl diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 2e1ccabf..9381d788 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -13,14 +13,21 @@ --out The measurement itself is unchanged — ``CosmologyValidation.calculate_2pcf`` -does the TreeCorr work and writes the ``.txt`` dump plus ξ+/ξ- FITS files into -``output_dir``. ``output_dir`` is passed explicitly (rather than via the +does the TreeCorr work and writes the ``.txt`` dump (a raw byproduct the +covariance machinery reads back). The analysis ξ± data product is then born as +SACC here: ``{ver}_xi_coarse.sacc``, a *part* on the coarse grid via +``xi_to_sacc(grid="coarse", ...)`` carrying ``theta_nom``/``npairs``/``weight`` +tags but NO covariance (the ξ block is supplied at assembly from the CosmoCov +theory covariance). ``output_dir`` is passed explicitly (rather than via the ``COSMO_VAL`` env hook) so lc can point each run at its own ``{output}`` tree. """ import argparse +import os +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.sacc_writers import xi_to_sacc def run_2pcf( @@ -31,30 +38,53 @@ def run_2pcf( npatch, cat_config, output_dir, - save_fits=True, ): - """Measure ξ±(θ) for ``ver`` and write it under ``output_dir``. + """Measure ξ±(θ) for ``ver`` and write its coarse SACC part under ``output_dir``. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides ``cat_config['paths']['output']`` so products land where lc expects. + + Returns + ------- + treecorr.GGCorrelation + The measured correlation object (also the source of the SACC part). """ cv = CosmologyValidation( versions=[ver], catalog_config=cat_config, output_dir=output_dir, ) - return cv.calculate_2pcf( + gg = cv.calculate_2pcf( ver=ver, npatch=npatch, - save_fits=save_fits, min_sep=min_sep, max_sep=max_sep, nbins=nbins, ) + # Born-as-SACC coarse ξ± part: no covariance here (added at assembly from + # the CosmoCov theory covariance). theta = meanr; theta_nom = rnom. + s = xi_to_sacc( + cv.sacc_nz(ver), + cv.sacc_metadata(ver), + gg.meanr, + gg.xip, + gg.xim, + grid="coarse", + theta_nom=gg.rnom, + npairs=gg.npairs, + weight=gg.weight, + ) + out_path = os.path.join( + output_dir or cv.cc["paths"]["output"], f"{ver}_xi_coarse.sacc" + ) + sacc_io.save(s, out_path) + print(f"Wrote coarse ξ± SACC part: {out_path}") + return gg + def _from_snakemake(smk): p = smk.params @@ -70,7 +100,6 @@ def _from_snakemake(smk): # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), - save_fits=True, ) @@ -97,7 +126,6 @@ def _from_cli(argv=None): "--cat-config", required=True, help="Absolute path to cat_config.yaml" ) ap.add_argument("--out", required=True, help="Output directory (lc {output})") - ap.add_argument("--no-fits", action="store_true", help="Skip ξ+/ξ- FITS export") a = ap.parse_args(argv) run_2pcf( ver=a.ver, @@ -107,7 +135,6 @@ def _from_cli(argv=None): npatch=a.npatch, cat_config=a.cat_config, output_dir=a.out, - save_fits=not a.no_fits, ) diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index eb31d2c7..ed3660bf 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -27,6 +27,12 @@ import treecorr from astropy.io import fits +# sacc_io depends only on numpy + sacc (no healpy/cs_util), so the born-as-SACC +# fine ξ± write works on the bare-host MPI path too, where the full cosmo_val +# stack is unavailable. +from sp_validation import sacc_io +from sp_validation.cosmo_val.sacc_writers import xi_to_sacc + try: # In-container path: full sp_validation stack available. from sp_validation.cosmo_val import CosmologyValidation @@ -76,6 +82,7 @@ E1_COL = None E2_COL = None W_COL = None +REDSHIFT_PATH = None # n(z) file for the SACC tracer TMIN = None # arcmin TMAX = None # arcmin NBINS = None @@ -196,35 +203,46 @@ def compute_patch_centers(ra, dec): del cat_sub -def write_xi_fits(gg, prefix, xi_data): - """Write ξ+ or ξ- to FITS matching CosmologyValidation format.""" - out_path = os.path.join( - OUTPUT_DIR, - f"{prefix}_{VERSION}_minsep={TMIN}_maxsep={TMAX}_nbins={NBINS}_npatch=1.fits", +def write_xi_fine_sacc(gg): + """Write the terminal fine-grid ξ± SACC part (``{version}_xi_fine.sacc``). + + This is a terminal product in its own right — COSEBIs and pure-E/B consume + it. It carries a ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim`` + (npatch=1 leaves shot-noise variance as the only covariance estimate). + Both run paths land here: in-container this uses the full SACC stack; on the + bare-host MPI run only ``sacc_io`` + the n(z) file are needed (no healpy). + """ + z, nz = np.loadtxt(REDSHIFT_PATH, unpack=True) + metadata = { + "catalogue_version": VERSION, + "sp_validation_version": _sp_validation_version(), + "npatch": 1, + } + s = xi_to_sacc( + {0: (z, nz)}, + metadata, + gg.meanr, + gg.xip, + gg.xim, + grid="fine", + theta_nom=gg.rnom, + variances=np.concatenate([gg.varxip, gg.varxim]), ) - n = len(xi_data) - cols = [ - fits.Column(name="BIN1", format="K", array=np.ones(n, dtype=int)), - fits.Column(name="BIN2", format="K", array=np.ones(n, dtype=int)), - fits.Column(name="ANGBIN", format="K", array=np.arange(1, n + 1)), - fits.Column(name="VALUE", format="D", array=xi_data), - fits.Column(name="ANG", format="D", unit="arcmin", array=gg.meanr), - ] - ext_name = "XI_PLUS" if "plus" in prefix else "XI_MINUS" - hdu = fits.BinTableHDU.from_columns(cols, name=ext_name) - for key, val in { - "2PTDATA": "T", - "QUANT1": "G+R", - "QUANT2": "G+R", - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - }.items(): - hdu.header[key] = val - hdu.writeto(out_path, overwrite=True) + out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_fine.sacc") + sacc_io.save(s, out_path) log(f" Wrote {out_path}") +def _sp_validation_version(): + """Best-effort package version for the SACC metadata (empty if unavailable).""" + try: + from sp_validation import __version__ + + return __version__ + except Exception: + return "" + + def resolve_shear_config(cat_config_path, version): """Standalone shear-config resolver (bare-host fallback for CosmologyValidation). @@ -274,7 +292,7 @@ def resolve_paths(ver): def main(): - global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL + global CAT_PATH, VERSION, E1_COL, E2_COL, W_COL, REDSHIFT_PATH global TMIN, TMAX, NBINS, NPATCH, OUTPUT_DIR, PATCH_FILE args = parse_args() @@ -301,6 +319,7 @@ def main(): E1_COL = shear_cfg["e1_col"] E2_COL = shear_cfg["e2_col"] W_COL = shear_cfg["w_col"] + REDSHIFT_PATH = shear_cfg["redshift_path"] PATCH_FILE = os.path.join( OUTPUT_DIR, @@ -383,8 +402,7 @@ def main(): gg.write(out_txt, write_patch_results=False, write_cov=False) log(f" Wrote {out_txt}") - write_xi_fits(gg, "xi_plus", gg.xip) - write_xi_fits(gg, "xi_minus", gg.xim) + write_xi_fine_sacc(gg) elapsed = time.time() - t0 log(f"Done! Total time: {elapsed / 3600:.1f}h ({elapsed:.0f}s)") From 700a88579942ac5afb6574c1949d6fa2a55fb85a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 07:34:24 +0200 Subject: [PATCH 12/38] =?UTF-8?q?fix(twopoint):=20fail=20fast=20on=20tomog?= =?UTF-8?q?raphic/=CE=BE-less=20SACC;=20pin=20covariance=20gather?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial-review hardening (one HIGH, one MEDIUM, one coverage gap): - HIGH: n_bins>1 silently truncated — n_bins drove only the NZDATA column count while data/covariance were read from bin (0, 0), so a 2-bin SACC emitted a plausible-looking FITS carrying 1/3 of the data. The converter now fails fast unless n_bins == 1 and the ξ tracer pairs are exactly {(source_0, source_0)}; tomographic emission lands with the tomographic round. - MEDIUM: a ξ-less SACC wrote an empty XI_PLUS and a (0, 0) COVMAT silently; now a loud ValueError. - Coverage: every prior covariance test exercised the identity permutation (single-pair SACC order is already type-major). A (row, col)-encoded covariance test now pins the exact np.ix_ gather of COVMAT (block-diag ξ + joint non-adjacent [τ0+; τ2+]) and COVMAT_CELL — any transposition, offset, or swapped block fails. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Ko89uZF84ez6HEDcTWbJe --- .../tests/test_twopoint_convert.py | 94 +++++++++++++++++++ src/sp_validation/twopoint_convert.py | 42 ++++++++- 2 files changed, 134 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py index 5e599717..3a5bc44a 100644 --- a/src/sp_validation/tests/test_twopoint_convert.py +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -376,3 +376,97 @@ def test_rho_tau_sidecars_required_together(tmp_path): twopoint_convert.sacc_to_twopoint_fits( s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 ) + + +# ============================================================================= +# Fail-fast guards and permutation teeth (adversarial-review hardening) +# ============================================================================= + + +def test_tomographic_sacc_raises(tmp_path): + """A multi-bin SACC fails fast instead of silently truncating to (0, 0). + + Review finding (HIGH): ``n_bins`` alone drove the NZDATA column count while + the data vector and covariance were read from bin ``(0, 0)`` only, so a + 2-bin SACC + ``n_bins=2`` emitted a plausible-looking FITS carrying 1/3 of + the data. Both the ``n_bins`` and the tracer-pair mismatch must raise. + """ + inp = _inputs(seed=30) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"]), 1: (inp["z"], inp["nz"])}) + for pair in [(0, 0), (0, 1), (1, 1)]: + sacc_io.add_xi(s, pair, inp["theta"], inp["xip"], inp["xim"], grid="coarse") + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + assert not (tmp_path / "x.fits").exists() + + +def test_sacc_without_xi_raises(tmp_path): + """A SACC with no ξ± points raises instead of writing an empty data vector.""" + inp = _inputs(seed=31) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="nothing to convert"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + assert not (tmp_path / "x.fits").exists() + + +def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): + """Every COVMAT/COVMAT_CELL entry is the exact ``np.ix_`` gather of the SACC + covariance, pinned with a (row, col)-encoded matrix. + + Review finding (MEDIUM): for a single bin pair the ξ gather happens to be + the identity permutation, so the byte-compares alone could pass with a + transposed or block-swapped gather. Encoding ``C[i, j] = i*n + j`` (asymmetric, + every entry unique) makes any transposition, offset, or wrong block produce + detectably wrong values; the τ gather is genuinely non-identity (τ_0− sits + between τ_0+ and τ_2+ in insertion order). Expected layout per + ``covdat_to_fits``: block_diag(ξ type-major gather, joint [τ_0+; τ_2+] + gather), with COVMAT_CELL the CELL_EE gather in its own HDU. + """ + inp = _inputs(seed=32) + s = _sacc(inp, cl=True, rho_tau=True) + n = len(s.mean) + encoded = np.arange(n * n, dtype=float).reshape(n, n) + s.add_covariance(encoded, overwrite=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + + out = tmp_path / "encoded.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + pair = (SOURCE, SOURCE) + xi_idx = np.concatenate( + [s.indices(sacc_io.XI_PLUS, pair), s.indices(sacc_io.XI_MINUS, pair)] + ) + tau_idx = np.concatenate( + [ + s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), + s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), + ] + ) + expected = twopoint_convert._block_diag( + encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] + ) + cell_idx = s.indices(sacc_io.CL_EE, pair) + + with fits.open(out) as hdul: + np.testing.assert_array_equal(hdul["COVMAT"].data, expected) + np.testing.assert_array_equal( + hdul["COVMAT_CELL"].data, encoded[np.ix_(cell_idx, cell_idx)] + ) diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py index c06e0eb3..8f787f3b 100644 --- a/src/sp_validation/twopoint_convert.py +++ b/src/sp_validation/twopoint_convert.py @@ -160,6 +160,33 @@ def _type_major_xi(s, bins): return sacc_io.get_xi(s, bins, grid="coarse") +def _require_single_bin(s, n_bins): + """Fail fast unless the SACC is a valid single-bin ξ product. + + The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads + (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip + through silently — ``n_bins`` alone drives the NZDATA column count while the + ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a + ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. + Guards both the empty-ξ case and the single-bin contract; tomographic + emission lands with the tomographic round. + """ + pairs = s.get_tracer_combinations(sacc_io.XI_PLUS) + if not pairs: + raise ValueError( + f"SACC has no {sacc_io.XI_PLUS} points — nothing to convert; the " + "2pt-FITS data vector is built from the ξ± statistics" + ) + expected = (sacc_io.source_name(0), sacc_io.source_name(0)) + if n_bins != 1 or set(pairs) != {expected}: + raise ValueError( + f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " + f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " + "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " + "bin) lands with the tomographic round." + ) + + def sacc_to_twopoint_fits( s, path, @@ -191,19 +218,30 @@ def sacc_to_twopoint_fits( alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When omitted, a pure ξ (± Cℓ) product is written. n_bins : int, optional - Number of source tomographic bins (default 1, the current single-bin - analysis). Sets the NZDATA column count. + Number of source tomographic bins. Must be ``1``: this converter emits + the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic + emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) + lands with the tomographic round; the converter fails fast on anything + else rather than silently truncating to bin ``(0, 0)``. Returns ------- astropy.io.fits.HDUList The assembled list, also written to ``path``. + + Raises + ------ + ValueError + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + pairs are anything other than exactly ``{(source_0, source_0)}`` (the + single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. """ if (rho_stats_hdu is None) != (tau_stats_hdu is None): raise ValueError( "rho_stats_hdu and tau_stats_hdu must be supplied together " "(the ρ/τ product needs both, or neither for a pure-ξ product)" ) + _require_single_bin(s, n_bins) use_rho_tau = rho_stats_hdu is not None bins = (0, 0) From 91f5d43e4fa72dfb8435134eabe2c2fdb6b8932f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 07:39:58 +0200 Subject: [PATCH 13/38] test(pseudo_cl): fixture carries shear.redshift_path like the real cat_config The migrated pseudo-Cl SACC part calls get_redshift(), which reads cc[version]['shear']['redshift_path']; the synthetic fixture wrote the dndz file but never pointed that key at it (KeyError at core.py:422). Verified: the KeyError layer is resolved; the end-to-end test now fails one layer deeper (test asserts the legacy ELL/EE/BB/EB FITS columns while the migrated writer emits the SACC part - the open wiring seam). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Ko89uZF84ez6HEDcTWbJe --- src/sp_validation/tests/test_pseudo_cl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 45a2366d..6fc43d55 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -112,6 +112,7 @@ def _write_synthetic_config(tmp_path): shear_cfg = { "path": "shear.fits", + "redshift_path": str(nz_dir / "dndz_SP_A.txt"), "w_col": "w", "e1_col": "e1", "e2_col": "e2", From dd474b3e17b28c11fd913824be00552eb580642d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 07:40:49 +0200 Subject: [PATCH 14/38] WIP(workflow): born-as-SACC rule wiring - UNVERIFIED, interrupted mid-flight Working-tree state of the wiring subagent when it died on the session limit (its final turn also went unreviewed by the safety classifier). Wires cosmo_val.smk/twopoint.smk rules and cv_* scripts to the SACC parts + assemble_sacc rule. NOT dry-run-tested (no snakemake in the shared venv), NOT reviewed; test_calculate_pseudo_cl_catalog_end_to_end still fails at the writer/consumer seam (test reads legacy ELL columns). Next session: verify this diff against the DAG design in the PR-4 plan, reconcile the pseudo-Cl end-to-end test with the born-as-SACC output, add writer-path tests, run the suite, then de-WIP. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Ko89uZF84ez6HEDcTWbJe --- workflow/rules/cosmo_val.smk | 40 +++++++++++++++++++++---- workflow/rules/twopoint.smk | 27 ++++++++++------- workflow/scripts/cv_cosebis.py | 18 +++++++++-- workflow/scripts/cv_pure_eb.py | 12 +++++--- workflow/scripts/cv_summarize_bmodes.py | 4 +-- 5 files changed, 76 insertions(+), 25 deletions(-) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 704b4fb0..000b445a 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -104,8 +104,36 @@ def cv_cosebis_npz(version): ) -def cv_pseudo_cl_fits(version): - return str(COSMO_VAL / f"pseudo_cl_{version}.fits") +def cv_pseudo_cl_sacc(version): + """Pseudo-Cl SACC part calculate_pseudo_cl writes (born as SACC).""" + return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") + + +def cv_cosebis_sacc(version): + """COSEBIs SACC part (fiducial scale cut) the cv_cosebis rule writes.""" + return str(COSMO_VAL / f"{version}_cosebis.sacc") + + +def cv_pure_eb_sacc(version): + """Pure-E/B SACC part the cv_pure_eb rule writes.""" + return str(COSMO_VAL / f"{version}_pure_eb.sacc") + + +def cv_rho_tau_sacc(version): + """ρ/τ SACC part calculate_rho_tau_stats writes (rho_tau_{base}.sacc).""" + return str( + COSMO_VAL / "rho_tau_stats" / f"rho_tau_{cv_basename(version, CV_FIDUCIAL)}.sacc" + ) + + +def cv_xi_coarse_sacc(version): + """Coarse ξ± SACC part the xi rule (run_2pcf.py) writes for a version.""" + return str(COSMO_VAL / f"{version}_xi_coarse.sacc") + + +def cv_analysis_sacc(version): + """Terminal assembled analysis file {version}.sacc.""" + return str(COSMO_VAL / f"{version}.sacc") # Common params block shared by every cosmo_val rule: the cv constructor kwargs @@ -275,9 +303,9 @@ rule cv_ratio_xi_sys_xi: # --------------------------------------------------------------------------- rule cv_pseudo_cl: - """Pseudo-Cl E/B spectra for all versions (NaMaster).""" + """Pseudo-Cl E/B spectra for all versions (NaMaster), born as SACC parts.""" output: - pseudo_cl=[cv_pseudo_cl_fits(v) for v in CV_VERSIONS], + pseudo_cl=[cv_pseudo_cl_sacc(v) for v in CV_VERSIONS], params: **cv_params(), threads: 12 @@ -298,6 +326,7 @@ rule cv_pure_eb: xi=lambda w: cv_xi_txt(w.version), output: npz=cv_pure_eb_npz("{version}"), + sacc=cv_pure_eb_sacc("{version}"), params: version="{version}", min_sep_int=CV["pure_eb"]["min_sep_int"], @@ -320,6 +349,7 @@ rule cv_cosebis: xi=lambda w: cv_xi_txt(w.version), output: npz=cv_cosebis_npz("{version}"), + sacc=cv_cosebis_sacc("{version}"), params: version="{version}", min_sep_int=CV["cosebis"]["min_sep_int"], @@ -345,7 +375,7 @@ rule cv_summarize_bmodes: pure_eb=[cv_pure_eb_npz(v) for v in CV_VERSIONS], cosebis=[cv_cosebis_npz(v) for v in CV_VERSIONS], pseudo_cl=( - [cv_pseudo_cl_fits(v) for v in CV_VERSIONS] + [cv_pseudo_cl_sacc(v) for v in CV_VERSIONS] if CV.get("include_pseudo_cl", False) else [] ), output: diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 22c09db2..1e23c6ce 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -5,9 +5,11 @@ rule xi: input: catalog=get_shear_catalog, output: - str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # Raw TreeCorr .txt byproduct (read back by covariance + skip-if-exists) + # and the born-as-SACC coarse ξ± part (a .part — no covariance until the + # assemble_sacc rule injects the CosmoCov block). + txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), + xi_coarse=str(COSMO_VAL / "{version}_xi_coarse.sacc"), threads: 24 params: ver="{version}", @@ -15,7 +17,6 @@ rule xi: max_sep="{max_sep}", nbins="{nbins}", npatch="{npatch}", - fits=False, resources: mem_mb=30000, disk_mb=20000, @@ -25,12 +26,16 @@ rule xi: rule xi_highres: - """High-resolution xi for COSEBIS integration.""" + """High-resolution xi for COSEBIS integration. + + Terminal born-as-SACC product: {version}_xi_fine.sacc (a DiagonalCovariance + from TreeCorr varxip/varxim). COSEBIs and pure-E/B consume it. The raw .txt + dump is kept as a convergence byproduct. + """ container: None output: txt=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.txt"), - xi_plus=str(COSMO_VAL / f"xi_plus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), - xi_minus=str(COSMO_VAL / f"xi_minus_{FIDUCIAL['version']}_minsep={FIDUCIAL['min_sep_int']}_maxsep={FIDUCIAL['max_sep_int']}_nbins=10000_npatch=1.fits"), + xi_fine=str(COSMO_VAL / f"{FIDUCIAL['version']}_xi_fine.sacc"), resources: tasks=30, cpus_per_task=12, @@ -92,9 +97,9 @@ wildcard_constraints: rule pseudo_cl: - """Generate pseudo-Cl data vector with configurable binning.""" + """Generate pseudo-Cl data vector (born as SACC) with configurable binning.""" output: - pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.fits"), + pseudo_cl=str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_{binning}_nbins={nbins}.sacc"), wildcard_constraints: blind="[ABC]", params: @@ -146,7 +151,7 @@ rule pseudo_cl_all: """Generate pseudo-Cls for all versions.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.fits"), + str(COSMO_VAL / "pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc"), version=PSEUDO_CL_VERSIONS, ), @@ -164,7 +169,7 @@ rule pseudo_cl_fine_all: """Generate fine pseudo-Cls for COSEBIS.""" input: expand( - str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.fits"), + str(COSMO_VAL / "pseudo_cl_{version}_blind={blind}_linear_nbins=2040.sacc"), version=config["versions"], blind=BLINDS, ), diff --git a/workflow/scripts/cv_cosebis.py b/workflow/scripts/cv_cosebis.py index 182cda99..b90ebc5b 100644 --- a/workflow/scripts/cv_cosebis.py +++ b/workflow/scripts/cv_cosebis.py @@ -3,8 +3,11 @@ Compute + plot rule (per version). plot_cosebis calls calculate_cosebis over a fine integration binning (the 2000-bin TreeCorr is the dominant cost) and evaluates the configured scale cuts. Writes the {version}_eb_..._data.npz -COSEBIs data product (declared output) plus figures, and the per-version -COSEBIs PTE that cv_summarize_bmodes collects. +COSEBIs data product plus figures, and the per-version COSEBIs PTE that +cv_summarize_bmodes collects. It also writes the born-as-SACC COSEBIs part +({version}_cosebis.sacc, the fiducial scale cut's {En,Bn,cov}) that the +assemble_sacc rule consumes — the multi-cut .npz sidecar stays the diagnostic +PTE scan. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs @@ -13,8 +16,9 @@ _unbuffer_streams() cv = make_cv(snakemake) p = snakemake.params +version = p["version"] cv.plot_cosebis( - version=p["version"], + version=version, min_sep_int=p["min_sep_int"], max_sep_int=p["max_sep_int"], nbins_int=p["nbins_int"], @@ -23,4 +27,12 @@ scale_cuts=[tuple(sc) for sc in p["scale_cuts"]], fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), ) +# Born-as-SACC COSEBIs part at the fiducial scale cut (plot_cosebis stored the +# multi-cut results on the instance). +cv.cosebis_to_sacc_part( + version, + snakemake.output["sacc"], + cv._cosebis_results[version], + fiducial_scale_cut=tuple(p["fiducial_scale_cut"]), +) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_pure_eb.py b/workflow/scripts/cv_pure_eb.py index d15a763f..7a453b16 100644 --- a/workflow/scripts/cv_pure_eb.py +++ b/workflow/scripts/cv_pure_eb.py @@ -3,9 +3,10 @@ Compute + plot rule (per version). plot_pure_eb calls calculate_pure_eb, which runs two TreeCorr correlations (reporting + integration binning); the reporting binning reuses the cv_2pcf data vector via calculate_2pcf's skip-if-exists -path. Writes the {version}_eb_..._data.npz data product (declared output) plus -companion figures, and the per-version E/B PTEs that cv_summarize_bmodes -collects. +path. Writes the {version}_eb_..._data.npz data product plus companion figures, +and the per-version E/B PTEs that cv_summarize_bmodes collects. It also writes +the born-as-SACC pure-E/B part ({version}_pure_eb.sacc, the six PURE_KEYS blocks ++ covariance) that the assemble_sacc rule consumes. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs @@ -14,12 +15,15 @@ _unbuffer_streams() cv = make_cv(snakemake) p = snakemake.params +version = p["version"] cv.plot_pure_eb( - versions=[p["version"]], + versions=[version], min_sep_int=p["min_sep_int"], max_sep_int=p["max_sep_int"], nbins_int=p["nbins_int"], fiducial_xip_scale_cut=tuple(p["fiducial_scale_cut"]), fiducial_xim_scale_cut=tuple(p["fiducial_scale_cut"]), ) +# Born-as-SACC pure-E/B part (plot_pure_eb stored the results on the instance). +cv.pure_eb_to_sacc_part(version, snakemake.output["sacc"], cv._pure_eb_results[version]) verify_outputs(snakemake) diff --git a/workflow/scripts/cv_summarize_bmodes.py b/workflow/scripts/cv_summarize_bmodes.py index 90df5999..0fe7b95a 100644 --- a/workflow/scripts/cv_summarize_bmodes.py +++ b/workflow/scripts/cv_summarize_bmodes.py @@ -3,8 +3,8 @@ The terminal diagnostic. summarize_bmodes reads the in-memory _pure_eb_results / _cosebis_results / _pseudo_cls dicts, which are populated by plot_pure_eb / plot_cosebis / plot_pseudo_cl. The per-version E/B and COSEBIs -npz products and the pseudo-Cl FITS are declared as inputs (so the DAG forces -those rules first), but the summary still needs the live result objects (it +npz products and the pseudo-Cl SACC parts are declared as inputs (so the DAG +forces those rules first), but the summary still needs the live result objects (it reads each version's TreeCorr `gg`, which the npz cannot hold). So this rule re-runs the three B-mode methods in-process: they reload the existing 2pcf / data-vector files via their skip-if-exists paths and recompute only the cheap From 02d9cf32d24c65c9daac1426e4a567f82e385099 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:01:31 +0200 Subject: [PATCH 15/38] test(pseudo_cl): read the born-as-SACC part in the end-to-end catalog test calculate_pseudo_cl_catalog is born-as-SACC (pseudo_cl_to_sacc_part writes EE/BB/EB + a shared bandpower window), so the end-to-end test can no longer read legacy ELL/EE/EB/BB FITS columns. Round-trip through sacc_io.get_pseudo_cl instead and assert the bandpower window rides the part per the layout contract. The pinned golden spectra are unchanged (identical computation, SACC serialization); they held bitwise through the round-trip. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/tests/test_pseudo_cl.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 6fc43d55..b7ac16b8 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -57,7 +57,9 @@ import pytest import yaml +from sp_validation import sacc_io from sp_validation.cosmo_val import CosmologyValidation +from sp_validation.cosmo_val.sacc_writers import BIN as SACC_BIN from sp_validation.rho_tau import get_params_rho_tau # These tests need the full harmonic-space stack (pymaster/NaMaster + healpy), @@ -509,24 +511,24 @@ def test_apply_random_rotation_reproducible_with_seed(cv, cat_and_params): # calculate_pseudo_cl_catalog -- deterministic end-to-end catalog path # =========================================================================== def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): - """End-to-end catalog path: FITS round-trip of ell + EE/EB/BB. + """End-to-end catalog path: SACC round-trip of ell + EE/EB/BB. The catalog method has no random noise debiasing, so it is reproducible to - the same ~2e-12 catalog-path float noise. save_pseudo_cl stores ELL/EE/EB/BB - (it drops the BE row); we pin the round-tripped table. + the same ~2e-12 catalog-path float noise. calculate_pseudo_cl_catalog is + born-as-SACC: it writes a pseudo-Cl part (EE/BB/EB + shared bandpower + window) via pseudo_cl_to_sacc_part; we pin the round-tripped spectra read + back through sacc_io.get_pseudo_cl. """ ver = cv._test_version cv._pseudo_cls = {ver: {}} - out_path = cv._output_path(f"pseudo_cl_cat_{ver}.fits") + out_path = cv._output_path(f"pseudo_cl_{ver}.sacc") cv.calculate_pseudo_cl_catalog(ver, out_path) assert os.path.exists(out_path) - d = fits.getdata(out_path) - # FITS gives big-endian f8; normalize for value comparison. - ell = np.asarray(d["ELL"], dtype=np.float64) - ee = np.asarray(d["EE"], dtype=np.float64) - eb = np.asarray(d["EB"], dtype=np.float64) - bb = np.asarray(d["BB"], dtype=np.float64) + s = sacc_io.load(out_path) + ell, ee, bb, eb, window = sacc_io.get_pseudo_cl(s, SACC_BIN) + # A shared BandpowerWindow rides the part per the SACC layout contract. + assert window is not None npt.assert_allclose( ell, From 0f4a6f2f23d8770ed0bbeed1e9629a83e1e3f3d5 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:10:51 +0200 Subject: [PATCH 16/38] fix(workflow): wire the assemble_sacc rule and the per-statistic SACC parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The born-as-SACC parts existed but nothing assembled them. Wire the DAG: per-statistic parts (xi_coarse, pseudo_cl, cosebis, pure_eb, rho_tau) → assemble_sacc rule → terminal {version}.sacc. - rule xi: give the coarse ξ± .sacc output the same reporting-binning wildcards as the .txt (Snakemake requires one wildcard set per rule; the bare {version}_xi_coarse.sacc name left the binning wildcards unbound and broke DAG resolution). run_2pcf.py writes the part to the declared output path. - rule rho_tau_stats: declare the rho_tau .sacc part (already written by calculate_rho_tau_stats) as a real output; run_rho_tau.py verifies it. - new rule assemble_sacc + assemble_sacc_all: load the five parts in canonical order via assemble_sacc.py, injecting a documented diagonal placeholder for the cov-less ξ/pseudo-Cℓ blocks (real CosmoCov/NaMaster covariance plugs into the same --xi-cov/--pseudo-cl-cov seam later — PR-3 converter territory). - cosmo_val_all: request the terminal {version}.sacc per version. - cv_pseudo_cl.py docstring: born-as-SACC, no longer FITS. Dry-runs clean for assemble_sacc_all, cosmo_val_all, bmode_summary.json, and the fine-grid xi_fine target. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- workflow/rules/cosmo_val.smk | 75 +++++++++++++++++++++++++++++++- workflow/rules/twopoint.smk | 14 ++++-- workflow/scripts/cv_pseudo_cl.py | 8 ++-- workflow/scripts/run_2pcf.py | 13 ++++-- workflow/scripts/run_rho_tau.py | 6 ++- 5 files changed, 103 insertions(+), 13 deletions(-) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 000b445a..03441872 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -127,8 +127,18 @@ def cv_rho_tau_sacc(version): def cv_xi_coarse_sacc(version): - """Coarse ξ± SACC part the xi rule (run_2pcf.py) writes for a version.""" - return str(COSMO_VAL / f"{version}_xi_coarse.sacc") + """Coarse ξ± SACC part the xi rule (run_2pcf.py) writes for a version. + + Carries the reporting-binning suffix so requesting it binds the xi job's + wildcards (the rule's txt + coarse .sacc outputs share one wildcard set). + """ + return str( + COSMO_VAL + / ( + f"{version}_xi_coarse_minsep={CV['theta_min']}_maxsep={CV['theta_max']}" + f"_nbins={CV['nbins']}_npatch={CV['npatch']}.sacc" + ) + ) def cv_analysis_sacc(version): @@ -401,6 +411,65 @@ rule cv_summarize_bmodes: "../scripts/cv_summarize_bmodes.py" +# --------------------------------------------------------------------------- +# Terminal analysis file: assemble the per-statistic SACC parts into {version}.sacc +# --------------------------------------------------------------------------- +# The five born-as-SACC parts (xi_coarse, pseudo_cl, cosebis, pure_eb, rho_tau) +# are each written by their own rule carrying its own covariance block, except +# ξ± coarse and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py +# loads the parts in canonical order and rebuilds one {version}.sacc with a +# single FullCovariance (point-insertion order = block order). The cov-less +# ξ/pseudo-Cℓ blocks are supplied here: the real CosmoCov / NaMaster covariances +# plug in via --xi-cov / --pseudo-cl-cov when the analysis needs them (that +# sourcing is the PR-3 converter's territory); until then a documented diagonal +# placeholder keeps the FullCovariance structurally valid. The placeholder is a +# flagged stand-in, never a science covariance. + + +def cv_assemble_inputs(version): + """The per-statistic SACC parts assemble_sacc consumes for a version. + + Each part's filename carries enough to bind its producing rule's wildcards + (the coarse ξ± part its reporting binning, the ρ/τ part likewise). pseudo_cl + is included only when the config toggles the harmonic-space BB into the + analysis. + """ + parts = dict( + xi_coarse=cv_xi_coarse_sacc(version), + cosebis=cv_cosebis_sacc(version), + pure_eb=cv_pure_eb_sacc(version), + rho_tau=cv_rho_tau_sacc(version), + ) + if CV.get("include_pseudo_cl", False): + parts["pseudo_cl"] = cv_pseudo_cl_sacc(version) + return parts + + +rule assemble_sacc: + """Assemble the terminal {version}.sacc from the per-statistic SACC parts.""" + input: + unpack(lambda w: cv_assemble_inputs(w.version)), + output: + sacc=cv_analysis_sacc("{version}"), + params: + version="{version}", + # Real ξ / pseudo-Cℓ covariance sourcing (CosmoCov / NaMaster) plugs in + # here later; for now a documented diagonal placeholder keeps the + # assembled FullCovariance structurally valid. + placeholder_var=1.0, + resources: + mem_mb=8000, + runtime=20, + script: + "../scripts/assemble_sacc.py" + + +rule assemble_sacc_all: + """Assemble the analysis SACC file for every version.""" + input: + [cv_analysis_sacc(v) for v in CV_VERSIONS], + + # --------------------------------------------------------------------------- # Aggregate target: the whole validation suite # --------------------------------------------------------------------------- @@ -422,3 +491,5 @@ rule cosmo_val_all: str(COSMO_VAL / "ratio_xi_sys_xi.png"), # B-modes str(COSMO_VAL / "bmode_summary.json"), + # Terminal analysis file: the assembled {version}.sacc per version + [cv_analysis_sacc(v) for v in CV_VERSIONS], diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index 1e23c6ce..c39553cb 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -6,10 +6,13 @@ rule xi: catalog=get_shear_catalog, output: # Raw TreeCorr .txt byproduct (read back by covariance + skip-if-exists) - # and the born-as-SACC coarse ξ± part (a .part — no covariance until the - # assemble_sacc rule injects the CosmoCov block). + # and the born-as-SACC coarse ξ± part (no covariance until the + # assemble_sacc rule injects the CosmoCov block). Both outputs carry the + # same reporting-binning wildcards — Snakemake requires every output of a + # rule to share one wildcard set, and it keeps the coarse .sacc name + # self-describing so requesting it binds the xi job unambiguously. txt=str(COSMO_VAL / "{version}_xi_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.txt"), - xi_coarse=str(COSMO_VAL / "{version}_xi_coarse.sacc"), + xi_coarse=str(COSMO_VAL / "{version}_xi_coarse_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 24 params: ver="{version}", @@ -75,6 +78,11 @@ rule rho_tau_stats: output: rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), + # Born-as-SACC ρ/τ part (ρ_0…ρ_5 autos + τ_0/τ_2/τ_5 leakage, carrying + # its own covariance block) that the assemble_sacc rule consumes; + # calculate_rho_tau_stats writes it alongside the FITS via + # rho_tau_to_sacc_part. + rho_tau=str(COSMO_VAL / "rho_tau_stats/rho_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.sacc"), threads: 48 params: ver="{version}", diff --git a/workflow/scripts/cv_pseudo_cl.py b/workflow/scripts/cv_pseudo_cl.py index cf04e8e8..43923e4e 100644 --- a/workflow/scripts/cv_pseudo_cl.py +++ b/workflow/scripts/cv_pseudo_cl.py @@ -1,8 +1,10 @@ """Rule cv_pseudo_cl: harmonic-space pseudo-Cl B-mode spectra. -plot_pseudo_cl triggers calculate_pseudo_cl, which writes pseudo_cl_{version}.fits -for every version (the BB spectrum cv_summarize_bmodes reads) and the cell_ee.png -figure. The per-version FITS files are the declared outputs. +plot_pseudo_cl triggers calculate_pseudo_cl, which writes the born-as-SACC +pseudo_cl_{version}.sacc part for every version (EE/BB/EB with the shared +bandpower window — the BB spectrum cv_summarize_bmodes reads) and the +cell_ee.png figure. The per-version SACC parts are the declared outputs and +feed both cv_summarize_bmodes and the assemble_sacc rule. """ from cv_runner import _unbuffer_streams, make_cv, verify_outputs diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index 9381d788..a773d039 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -38,14 +38,18 @@ def run_2pcf( npatch, cat_config, output_dir, + sacc_out=None, ): - """Measure ξ±(θ) for ``ver`` and write its coarse SACC part under ``output_dir``. + """Measure ξ±(θ) for ``ver`` and write its coarse SACC part. Parameters mirror the TreeCorr reporting/integration grids: ``min_sep`` / ``max_sep`` in arcmin, ``nbins`` logarithmic bins, ``npatch`` spatial patches (1 for the paper fiducial). ``cat_config`` is an absolute path to the catalog configuration; ``output_dir`` overrides - ``cat_config['paths']['output']`` so products land where lc expects. + ``cat_config['paths']['output']`` so the ``.txt`` byproduct lands where lc + expects. ``sacc_out`` is the exact destination for the coarse ξ± SACC part + (the Snakemake-declared output); it defaults to ``{ver}_xi_coarse.sacc`` + under the resolved output directory for the CLI path. Returns ------- @@ -78,7 +82,7 @@ def run_2pcf( npairs=gg.npairs, weight=gg.weight, ) - out_path = os.path.join( + out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], f"{ver}_xi_coarse.sacc" ) sacc_io.save(s, out_path) @@ -100,6 +104,9 @@ def _from_snakemake(smk): # class defaults (./cat_config.yaml, COSMO_VAL env) otherwise. cat_config=p.get("cat_config", "./cat_config.yaml"), output_dir=p.get("output_dir", None), + # Write the SACC part exactly where the rule declares it (the .txt + # byproduct still lands under the resolved output dir via _output_path). + sacc_out=smk.output["xi_coarse"], ) diff --git a/workflow/scripts/run_rho_tau.py b/workflow/scripts/run_rho_tau.py index ea2f35bc..9df0bbfd 100644 --- a/workflow/scripts/run_rho_tau.py +++ b/workflow/scripts/run_rho_tau.py @@ -48,9 +48,11 @@ cv.calculate_rho_tau_stats() -# Confirm CosmologyValidation produced the requested outputs +# Confirm CosmologyValidation produced the requested outputs. calculate_rho_tau_stats +# writes the rho/tau FITS *and* the born-as-SACC rho_tau part (via +# rho_tau_to_sacc_part); the part feeds the assemble_sacc rule. outputs = snakemake.output # type: ignore -for label in ("rho_stats", "tau_stats"): +for label in ("rho_stats", "tau_stats", "rho_tau"): target = Path(outputs[label]) if not target.exists(): raise FileNotFoundError( From 99db96e14d8c5ce439051197bc3275d6efb64411 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:14:08 +0200 Subject: [PATCH 17/38] test(assemble_sacc): integration test for the assemble_sacc.py script seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pure assembler is covered in test_sacc_writers; this pins the DAG-facing script that loads per-statistic .sacc part *files* and rebuilds {version}.sacc: canonical block order across all five statistics, the diagonal placeholder for the cov-less ξ± part, real CosmoCov ξ covariance .txt injection, the fail-loud path when a required covariance is absent, and the pseudo_cl config toggle. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/tests/test_assemble_sacc.py | 206 ++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 src/sp_validation/tests/test_assemble_sacc.py diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py new file mode 100644 index 00000000..a14d3f06 --- /dev/null +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -0,0 +1,206 @@ +"""Integration tests for the ``assemble_sacc.py`` workflow script. + +The pure assembler (``sacc_writers.assemble_analysis_sacc``) is covered in +``test_sacc_writers.py``. This file exercises the *script seam* the DAG uses: +``assemble_sacc.assemble_sacc`` loads per-statistic ``.sacc`` part *files* in +CANONICAL order, injects the born-cov-less ξ± / pseudo-Cℓ blocks (real CosmoCov +/ NaMaster covariance, or a flagged diagonal placeholder), and writes one +``{version}.sacc`` whose points and covariance blocks land in canonical order. + +The script lives under ``workflow/scripts`` (off the package path); it is loaded +by file path exactly as the lightcone/ASTRA CLI path imports it. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio +from sp_validation.cosmo_val import sacc_writers as sw + + +def _load_assemble_module(): + """Import ``workflow/scripts/assemble_sacc.py`` by file path.""" + repo_root = next( + p for p in Path(__file__).resolve().parents if (p / "pyproject.toml").exists() + ) + path = repo_root / "workflow" / "scripts" / "assemble_sacc.py" + spec = importlib.util.spec_from_file_location("assemble_sacc", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +asm = _load_assemble_module() + + +def _nz(seed=0, n=40): + rng = np.random.default_rng(seed) + return np.linspace(0.01, 2.0, n), rng.uniform(0.1, 1.0, n) + + +def _spd(n, seed): + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _theta(n=6): + return np.geomspace(1.0, 100.0, n) + + +META = {"catalogue_version": "vSYNTH", "npatch": 1} + + +def _write_parts(tmp_path, *, with_pseudo_cl=True, cov_less=("xi_coarse",)): + """Write per-statistic parts to disk; return the ``{name: path}`` mapping. + + Parts named in ``cov_less`` are written without a covariance (mimicking the + born-cov-less ξ± coarse / pseudo-Cℓ parts); the rest carry their own block. + """ + nz = {0: _nz()} + theta = _theta() + ell = np.array([30.0, 60.0, 90.0]) + + class _Wsp: + def get_bandpower_windows(self): + w = np.zeros((4, 3, 4, 20)) + for out in range(4): + for b in range(3): + w[out, b, out, b * 6 : b * 6 + 6] = 1.0 + return w + + xi = sw.xi_to_sacc( + nz, META, theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + if "xi_coarse" not in cov_less: + xi.add_covariance(_spd(len(xi.mean), 1)) + + cl_all = np.vstack( + [np.arange(3) * 1e-9, np.arange(3) * 2e-9, np.zeros(3), np.arange(3) * 3e-9] + ) + cl = sw.pseudo_cl_to_sacc( + nz, + META, + ell, + cl_all, + _Wsp(), + covariance=None if "pseudo_cl" in cov_less else _spd(9, 2), + ) + + co = sw.cosebis_to_sacc( + nz, + META, + { + "En": np.arange(1, 6) * 1e-6, + "Bn": np.arange(1, 6) * 1e-7, + "cov": _spd(10, 3), + }, + (1.0, 100.0), + ) + + eb_arrays = { + key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS) + } + eb = sw.pure_eb_to_sacc(nz, META, theta, eb_arrays, covariance=_spd(36, 4)) + + rho = {"theta": theta} + tau = {"theta": theta} + rng = np.random.default_rng(5) + for k in sw.RHO_K: + for suffix in ("p", "m"): + rho[f"rho_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 + rho[f"varrho_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) + for k in sw.TAU_K: + for suffix in ("p", "m"): + tau[f"tau_{k}_{suffix}"] = rng.normal(size=6) * 1e-6 + tau[f"vartau_{k}_{suffix}"] = rng.uniform(1e-14, 1e-13, 6) + rt = sw.rho_tau_to_sacc(nz, META, rho, tau) + + parts = { + "xi_coarse": xi, + "pseudo_cl": cl, + "cosebis": co, + "pure_eb": eb, + "rho_tau": rt, + } + if not with_pseudo_cl: + parts.pop("pseudo_cl") + + paths = {} + for name, part in parts.items(): + p = tmp_path / f"{name}.sacc" + sio.save(part, str(p)) + paths[name] = str(p) + return paths + + +def test_assemble_sacc_placeholder_canonical_order(tmp_path): + """The cov-less ξ± part gets a placeholder; every point is covered and the + blocks land in canonical order (ξ±, pseudo-Cℓ, COSEBIs, pure-E/B, ρ, τ).""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + assert out.exists() + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + # Canonical insertion order: the first data types are ξ+ then ξ−. + types_in_order = [dp.data_type for dp in s.data] + assert types_in_order[0] == sio.XI_PLUS + assert sio.XI_MINUS in types_in_order + # ξ appears before pseudo-Cℓ before COSEBIs before pure-E/B before ρ/τ. + first = {t: types_in_order.index(t) for t in set(types_in_order)} + assert first[sio.XI_PLUS] < first[sio.CL_EE] < first[sio.COSEBI_EE] + assert first[sio.COSEBI_EE] < first[sio.PURE_TYPES["xip_E"]] + assert first[sio.PURE_TYPES["xip_E"]] < first[sio.RHO_PLUS.format(k=0)] + assert first[sio.RHO_PLUS.format(k=0)] < first[sio.TAU_PLUS.format(k=0)] + + # The ξ± block is the placeholder diagonal (variance 1.0 on its own points). + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + dense = s.covariance.dense + assert np.allclose(np.diag(dense[np.ix_(xi_idx, xi_idx)]), 1.0) + # ...and it does not bleed into the neighbouring COSEBIs block (cross zero). + co_idx = np.concatenate( + [s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)] + ) + assert np.allclose(dense[np.ix_(xi_idx, co_idx)], 0.0) + + +def test_assemble_sacc_injects_real_xi_covariance(tmp_path): + """A CosmoCov ξ covariance .txt is loaded into the cov-less ξ± block.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + # ξ± part has 12 points ([ξ+; ξ−] over 6 θ); supply a matching cov .txt. + xi_cov = _spd(12, 21) + cov_path = tmp_path / "xi_cov.txt" + np.savetxt(str(cov_path), xi_cov) + + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), xi_cov=str(cov_path)) + tr = ("source_0", "source_0") + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + assert np.allclose(s.covariance.dense[np.ix_(xi_idx, xi_idx)], xi_cov) + + +def test_assemble_sacc_missing_cov_raises(tmp_path): + """A cov-less part with no injected block and no placeholder fails loudly.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="carries no covariance"): + asm.assemble_sacc("vSYNTH", paths, str(out)) + + +def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): + """With pseudo_cl absent, assembly still succeeds and omits the Cℓ points.""" + paths = _write_parts(tmp_path, with_pseudo_cl=False, cov_less=("xi_coarse",)) + assert "pseudo_cl" not in paths + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc("vSYNTH", paths, str(out), placeholder_var=1.0) + tr = ("source_0", "source_0") + assert len(s.indices(sio.CL_EE, tr)) == 0 + # Round-trips as a valid FullCovariance over the remaining points. + s2 = sio.load(str(out)) + assert type(s2.covariance).__name__ == "FullCovariance" + assert s2.covariance.dense.shape == (len(s2.mean), len(s2.mean)) From 9afaeaa5d127d5fbb4ddc28b87a327762db5e24b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:24:56 +0200 Subject: [PATCH 18/38] fix(workflow): resolve xi_highres script path from the running checkout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MPI xi_highres rule shells out to run_2pcf_highres.py by absolute path through the deprecated pure_eb/ compat symlink (it can't use Snakemake's script: directive — mpiexec wraps the python call). Anchor the path on common.py's own location (WORKFLOW_SCRIPTS), which resolves to the generic workflow/scripts of whatever checkout parses the DAG, regardless of which paper composes it — workflow.basedir is unreliable here (under module composition it reflects the composing paper, verified: it resolved to papers/cosmo_val/scripts). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- workflow/common.py | 10 ++++++++++ workflow/rules/twopoint.smk | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/workflow/common.py b/workflow/common.py index 3df3413b..17e60ce7 100644 --- a/workflow/common.py +++ b/workflow/common.py @@ -5,6 +5,16 @@ import re from pathlib import Path +# Absolute path to the generic workflow's scripts, anchored on this module's own +# location (common.py lives in workflow/, is `from common import *`'d into every +# Snakefile, and so resolves to the generic workflow dir of the running checkout +# regardless of which paper composes it — unlike workflow.basedir, which under +# `module` composition reflects the composing paper). Rules that shell out to a +# script directly (the MPI xi_highres run can't go through Snakemake's `script:` +# directive) interpolate this instead of a hardcoded pure_eb/ compat-symlink +# path. /automnt/n17data is the automount of the container-bound /n17data. +WORKFLOW_SCRIPTS = os.path.join(os.path.dirname(os.path.realpath(__file__)), "scripts") + # Output roots are env-overridable so a reproduction run can write into a # fresh tree without clobbering (or silently reusing) prior products. COSMO_VAL = Path( diff --git a/workflow/rules/twopoint.smk b/workflow/rules/twopoint.smk index c39553cb..ebd19f91 100644 --- a/workflow/rules/twopoint.smk +++ b/workflow/rules/twopoint.smk @@ -1,4 +1,6 @@ # Two-point data-vector rules: xi, rho/tau, and pseudo-Cl products. +# WORKFLOW_SCRIPTS (from common.py) is the generic workflow's scripts dir, +# resolved from the running checkout — used by the raw-shell MPI xi_highres rule. rule xi: @@ -53,7 +55,7 @@ rule xi_highres: "--bind /home,/n09data,/n17data,/n23data1,/softs " "--env LD_LIBRARY_PATH=/softs/openmpi/5.0.5-slurm-CentOS8/lib " "/n17data/cdaley/containers/containers " - "python /automnt/n17data/cdaley/unions/pure_eb/code/sp_validation/workflow/scripts/run_2pcf_highres.py" + f"python {WORKFLOW_SCRIPTS}/run_2pcf_highres.py" rule run_cosmo_val: From 00460d858fbc7f2d21fc777ba4b08459d72185d1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:25:14 +0200 Subject: [PATCH 19/38] fix(workflow): assemble the analysis file from the tagged pseudo-Cl + real cov MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The analysis {version}.sacc must be byte-comparable against today's cosmosis_fitting.py assembly (the SACC layout contract's stated requirement for PR-3's converter). cosmosis consumes the tagged, blinded pseudo-Cl product (blind=A, powspace, nbins=32 per config harmonic.fiducial), not the untagged cv_pseudo_cl diagnostic — so assemble_sacc now inputs the tagged part and injects its real NaMaster covariance from the matching pseudo_cl_cov FITS (COVAR_EE_EE/BB_BB/EB_EB → block-diagonal). The untagged cv_pseudo_cl part stays the cv_summarize_bmodes B-mode diagnostic, unchanged. The ξ± coarse block keeps a documented diagonal placeholder: its real CosmoCov covariance is blind/gaussian/mask-keyed in the inference tree, and wiring it couples cosmo_val to the whole inference covariance DAG — that sourcing is PR-3 converter territory, ready at the --xi-cov seam. test_assemble_sacc: add the pseudo-Cl COVAR-FITS injection path (the live default) alongside the existing ξ .txt and placeholder paths. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/tests/test_assemble_sacc.py | 36 ++++++++++ workflow/rules/cosmo_val.smk | 68 +++++++++++++++---- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index a14d3f06..a43747dc 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -184,6 +184,42 @@ def test_assemble_sacc_injects_real_xi_covariance(tmp_path): assert np.allclose(s.covariance.dense[np.ix_(xi_idx, xi_idx)], xi_cov) +def test_assemble_sacc_injects_pseudo_cl_covariance(tmp_path): + """The NaMaster cov FITS (COVAR_EE_EE/BB_BB/EB_EB) → block-diagonal pseudo-Cℓ + block (the live default: ξ± placeholder + real pseudo-Cℓ cov).""" + from astropy.io import fits + + paths = _write_parts(tmp_path, cov_less=("xi_coarse", "pseudo_cl")) + # pseudo-Cℓ part is 3 ell × {EE, BB, EB} = 9 points; per-spectrum 3×3 blocks. + ee, bb, eb = _spd(3, 31), _spd(3, 32), _spd(3, 33) + cov_fits = tmp_path / "pseudo_cl_cov.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + fits.ImageHDU(ee, name="COVAR_EE_EE"), + fits.ImageHDU(bb, name="COVAR_BB_BB"), + fits.ImageHDU(eb, name="COVAR_EB_EB"), + ] + ).writeto(str(cov_fits)) + + out = tmp_path / "vSYNTH.sacc" + s = asm.assemble_sacc( + "vSYNTH", paths, str(out), pseudo_cl_cov=str(cov_fits), placeholder_var=1.0 + ) + tr = ("source_0", "source_0") + cl_idx = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + dense = s.covariance.dense + expected = np.zeros((9, 9)) + expected[0:3, 0:3], expected[3:6, 3:6], expected[6:9, 6:9] = ee, bb, eb + assert np.allclose(dense[np.ix_(cl_idx, cl_idx)], expected) + # ξ± stays the placeholder; the two blocks don't bleed into each other. + xi_idx = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + assert np.allclose(np.diag(dense[np.ix_(xi_idx, xi_idx)]), 1.0) + assert np.allclose(dense[np.ix_(xi_idx, cl_idx)], 0.0) + + def test_assemble_sacc_missing_cov_raises(tmp_path): """A cov-less part with no injected block and no placeholder fails loudly.""" paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 03441872..3fe0e1df 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -105,10 +105,38 @@ def cv_cosebis_npz(version): def cv_pseudo_cl_sacc(version): - """Pseudo-Cl SACC part calculate_pseudo_cl writes (born as SACC).""" + """Untagged pseudo-Cl SACC part cv_pseudo_cl writes (B-mode diagnostic). + + This is the harmonic-space BB diagnostic cv_summarize_bmodes reads. The + *analysis* file's pseudo-Cl part is the tagged, blinded inference product + instead (see cv_pseudo_cl_analysis_sacc) so {version}.sacc stays byte- + comparable against today's cosmosis_fitting.py assembly (PR-3's converter). + """ return str(COSMO_VAL / f"pseudo_cl_{version}.sacc") +# Fiducial harmonic-binning tag the pseudo-Cl producer (twopoint.smk rules +# pseudo_cl / pseudo_cl_cov) stamps into the analysis-grade filename. Mirrors +# inference.smk's PSEUDO_CL_TAG so the analysis file carries the same pseudo-Cl +# the inference pipeline consumes (canonical: blind=A, powspace, nbins=32). +_HARMONIC_FIDUCIAL = config["harmonic"]["fiducial"] +_PSEUDO_CL_TAG = ( + f"blind={_HARMONIC_FIDUCIAL['blind']}" + f"_{_HARMONIC_FIDUCIAL['binning']}" + f"_nbins={_HARMONIC_FIDUCIAL['nbins']}" +) + + +def cv_pseudo_cl_analysis_sacc(version): + """Tagged, blinded pseudo-Cl SACC part the analysis file carries.""" + return str(COSMO_VAL / f"pseudo_cl_{version}_{_PSEUDO_CL_TAG}.sacc") + + +def cv_pseudo_cl_cov(version): + """NaMaster pseudo-Cl covariance FITS (COVAR_EE_EE/BB_BB/EB_EB extensions).""" + return str(COSMO_VAL / f"pseudo_cl_cov_{version}_{_PSEUDO_CL_TAG}.fits") + + def cv_cosebis_sacc(version): """COSEBIs SACC part (fiducial scale cut) the cv_cosebis rule writes.""" return str(COSMO_VAL / f"{version}_cosebis.sacc") @@ -418,21 +446,29 @@ rule cv_summarize_bmodes: # are each written by their own rule carrying its own covariance block, except # ξ± coarse and pseudo-Cℓ which are born cov-less by design. assemble_sacc.py # loads the parts in canonical order and rebuilds one {version}.sacc with a -# single FullCovariance (point-insertion order = block order). The cov-less -# ξ/pseudo-Cℓ blocks are supplied here: the real CosmoCov / NaMaster covariances -# plug in via --xi-cov / --pseudo-cl-cov when the analysis needs them (that -# sourcing is the PR-3 converter's territory); until then a documented diagonal -# placeholder keeps the FullCovariance structurally valid. The placeholder is a -# flagged stand-in, never a science covariance. +# single FullCovariance (point-insertion order = block order). +# +# The pseudo-Cℓ part is the TAGGED, blinded inference product (blind=A, powspace, +# nbins=32) — the same pseudo-Cℓ today's cosmosis_fitting.py consumes — so the +# analysis file stays byte-comparable against it (PR-3's converter). Its real +# NaMaster covariance is injected here from the matching pseudo_cl_cov FITS +# (COVAR_EE_EE/BB_BB/EB_EB → block-diagonal, dropping cross-spectra, matching the +# B-mode PTE's use of COVAR_BB_BB). The ξ± coarse block is the one piece not yet +# sourced from its real covariance: the CosmoCov theory .txt is blind/gaussian/ +# mask-keyed and lives deep in the inference tree, so wiring it couples cosmo_val +# to the whole inference covariance DAG — that sourcing is PR-3's converter +# territory. Until then a documented diagonal placeholder keeps the ξ block (and +# so the FullCovariance) structurally valid; it is a flagged stand-in, never a +# science covariance, and plugs out via --xi-cov the moment PR 3 lands. def cv_assemble_inputs(version): - """The per-statistic SACC parts assemble_sacc consumes for a version. + """The per-statistic SACC parts + covariance inputs assemble_sacc consumes. Each part's filename carries enough to bind its producing rule's wildcards - (the coarse ξ± part its reporting binning, the ρ/τ part likewise). pseudo_cl - is included only when the config toggles the harmonic-space BB into the - analysis. + (the coarse ξ± and ρ/τ parts their reporting binning; the pseudo-Cℓ part its + fiducial harmonic tag). pseudo_cl (+ its cov) is included only when the + config toggles the harmonic-space BB into the analysis. """ parts = dict( xi_coarse=cv_xi_coarse_sacc(version), @@ -441,7 +477,8 @@ def cv_assemble_inputs(version): rho_tau=cv_rho_tau_sacc(version), ) if CV.get("include_pseudo_cl", False): - parts["pseudo_cl"] = cv_pseudo_cl_sacc(version) + parts["pseudo_cl"] = cv_pseudo_cl_analysis_sacc(version) + parts["pseudo_cl_cov"] = cv_pseudo_cl_cov(version) return parts @@ -453,9 +490,10 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - # Real ξ / pseudo-Cℓ covariance sourcing (CosmoCov / NaMaster) plugs in - # here later; for now a documented diagonal placeholder keeps the - # assembled FullCovariance structurally valid. + # ξ± coarse block: documented diagonal placeholder until PR-3's converter + # sources the real CosmoCov theory covariance via --xi-cov. The pseudo-Cℓ + # block is real (pseudo_cl_cov input); COSEBIs / pure-E/B / ρ/τ carry + # their own. assemble_sacc.py reads pseudo_cl_cov's COVAR_* extensions. placeholder_var=1.0, resources: mem_mb=8000, From 2d1747b6839690d6b80fc555a85fb47cb6c1ffba Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:26:11 +0200 Subject: [PATCH 20/38] test(workflow): dry-run guard uses sys.executable, drops SNAKEMAKE_PROFILE test_bmodes_workflow_dry_runs invoked a bare python3.12 (resolves off PATH to intel-python without snakemake) and inherited the login shell's slurm SNAKEMAKE_PROFILE (forces an executor plugin the test env need not have). Use sys.executable (the interpreter pytest/snakemake live in) and drop the profile for the dry-run. The guard now passes and exercises the migrated rule graph (rule xi's coarse-sacc output, the WORKFLOW_SCRIPTS path). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- .../tests/test_bmodes_workflow_dry_run.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 68c09981..60675f52 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -6,6 +6,7 @@ import os import subprocess +import sys from pathlib import Path import pytest @@ -31,11 +32,18 @@ def test_bmodes_workflow_dry_runs(): """The paper B-mode workflow must still parse and dry-run cleanly.""" workflow_dir = _repo_root() / "papers/bmodes" # PYTHONUNBUFFERED satisfies the Snakefile's `envvars:` declaration without - # depending on the invoking shell's environment. + # depending on the invoking shell's environment. A dry run resolves the DAG + # only — it never dispatches jobs — so drop any inherited SNAKEMAKE_PROFILE + # (e.g. the login shell's "slurm" profile), which would otherwise force an + # executor plugin the test environment need not have installed. env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} + env.pop("SNAKEMAKE_PROFILE", None) result = subprocess.run( [ - "python3.12", + # Invoke snakemake through the interpreter running the test — a bare + # "python3.12" resolves off PATH (e.g. intel-python without snakemake); + # sys.executable is the environment that pytest, hence snakemake, lives in. + sys.executable, "-m", "snakemake", "all_tapestry", From a592bb8350d7d9761d75d9f79e1e10e00feabc9a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 08:31:26 +0200 Subject: [PATCH 21/38] docs(inference): mark inference_prep inputs pre-SACC, migration deferred to PR 7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SACC migration removed the xi_plus/xi_minus FITS and switched pseudo_cl to .sacc, so inference_prep's DAG no longer resolves. It is not reachable from the cosmo_val suite (cosmo_val_all never requests it), so the cosmo_val DAG stays clean. Per the PR-4 scope this subsystem is left dormant — a comment block at the rule head names the stale inputs and states PR 7 (native-SACC inference consumption) rewires it to consume the assembled {version}.sacc directly, retiring cosmosis_fitting.py's per-product FITS assembly. Comment-only; no behavior change. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- workflow/rules/inference.smk | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index a93c0e71..6bf68645 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -55,11 +55,26 @@ def pseudo_cl_assets(version): cov_path = PSEUDO_CL_DIR / f"pseudo_cl_cov_{version}_{PSEUDO_CL_TAG}.fits" return str(cl_path), str(cov_path) +# --------------------------------------------------------------------------- +# DORMANT — pre-SACC cosmosis assembly. Migration to native SACC deferred to +# PR 7 (native-SACC inference consumption); do NOT deep-migrate here. +# +# The SACC migration (PR 4) removed the data products several of these inputs +# name, so this rule's DAG no longer resolves and is NOT reachable from the +# cosmo_val suite (cosmo_val_all never requests it). Stale inputs: +# - xi_plus / xi_minus FITS: the `xi` rule now emits the coarse ξ± SACC part +# ({version}_xi_coarse_...sacc), not per-sign FITS. +# - pseudo_cl / pseudo_cl_cov via pseudo_cl_assets(): the `pseudo_cl` rule now +# writes .sacc (pseudo_cl_assets still requests .fits). +# PR 7 rewires this to consume the assembled {version}.sacc (built by +# cosmo_val.smk's assemble_sacc rule) directly, retiring cosmosis_fitting.py's +# per-product FITS assembly. Until then the inference target is knowingly red. +# --------------------------------------------------------------------------- rule inference_prep: input: # Processed covariance matrix - use centralized covariance_path() cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), - # Xi FITS files + # Xi FITS files — PRE-SACC (no longer produced; see dormant note above) xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), # n(z) file (using new location with base version mapping) @@ -69,6 +84,7 @@ rule inference_prep: tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), # tau covariance (tracked as dependency) tau_cov=str(COSMO_VAL / "rho_tau_stats/cov_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_th.npy"), + # pseudo_cl / pseudo_cl_cov — PRE-SACC (.fits path; producer now writes .sacc) pseudo_cl=lambda w: pseudo_cl_assets(w.version)[0], pseudo_cl_cov=lambda w: pseudo_cl_assets(w.version)[1], output: From 60b74c60f6bc08df07e3156bfa89bdaf614e87c1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:19:14 +0200 Subject: [PATCH 22/38] fix(pseudo_cl): born-at-declared-name kills the cross-rule pseudo-Cl collision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both pseudo-Cl rules call calculate_pseudo_cl, which hardcoded the untagged native pseudo_cl_{ver}.sacc; generate_pseudo_cl.py then renamed it to the tagged name. In one cosmo_val_all DAG both fire: if the untagged cv_pseudo_cl (blind= None) ran first, the tagged rule's (blind=A) skip-if-exists silently adopted the blind=None file and the rename deleted cv_pseudo_cl's declared output — rebuild loops, and wrong-blind n(z) stamped into the terminal {version}.sacc's pseudo-Cl part. Thread out_path through calculate_pseudo_cl so each part is born directly at its final declared name (tagged for the producer, untagged for the diagnostic). No shared native basename, no rename; skip-if-exists keys on the declared path, so the two rules' paths are provably disjoint and the blind that computes each file matches its name. Multi-version + out_path now raises. generate_pseudo_cl.py and its CLI updated to pass the declared/native out_path directly. Regression tests: out_path is honoured (native name untouched) and the multi-version guard fires. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/cosmo_val/pseudo_cl.py | 45 +++++++++++++++------ src/sp_validation/tests/test_pseudo_cl.py | 29 ++++++++++++++ workflow/scripts/generate_pseudo_cl.py | 48 ++++++++++++----------- 3 files changed, 87 insertions(+), 35 deletions(-) diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 52d82849..29ff7846 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -454,20 +454,37 @@ def calculate_pseudo_cl_g_ng_cov(self, gaussian_part="iNKA"): f"Done Gaussian and Non-Gaussian covariance of the Pseudo-Cl's using {gaussian_part} for the Gaussian part" ) - def calculate_pseudo_cl(self): + def calculate_pseudo_cl(self, out_path=None): """ Compute the pseudo-Cl of given catalogs. - Each version's spectra are born as a SACC part (``pseudo_cl_{ver}.sacc``) - via :func:`sacc_writers.pseudo_cl_to_sacc` — EE/BB/EB carrying the shared - NaMaster bandpower window. The in-memory ``self._pseudo_cls[ver]`` - ``"pseudo_cl"`` entry keeps the ``ELL``/``EE``/``EB``/``BB`` arrays the - plotting and B-mode-summary consumers read by column name. + Each version's spectra are born as a SACC part via + :func:`sacc_writers.pseudo_cl_to_sacc` — EE/BB/EB carrying the shared + NaMaster bandpower window, with this instance's (blinded) n(z) stamped + in. The in-memory ``self._pseudo_cls[ver]`` ``"pseudo_cl"`` entry keeps + the ``ELL``/``EE``/``EB``/``BB`` arrays the plotting and B-mode-summary + consumers read by column name. + + ``out_path`` is the exact destination the part is *born at* — the + Snakemake-declared output. It must resolve per version; single-version + rules (the tagged blinded producer) pass their tagged output directly. + When ``None`` (multi-version diagnostic / the ``pseudo_cls`` property) + each part defaults to the untagged native ``pseudo_cl_{ver}.sacc``. + Skip-if-exists keys on this final path, so no two rules ever share an + undeclared native basename (a tagged product born at its native name and + then renamed would let one rule's skip-if-exists silently adopt — and + the rename delete — another rule's declared, differently-blinded file). """ self.print_start("Computing pseudo-Cl's") nside = self.nside + if out_path is not None and len(self.versions) != 1: + raise ValueError( + "calculate_pseudo_cl(out_path=...) writes one part to one path, " + f"but {len(self.versions)} versions are configured; call per version" + ) + try: self._pseudo_cls except AttributeError: @@ -477,14 +494,18 @@ def calculate_pseudo_cl(self): self._pseudo_cls[ver] = {} - out_path = self._output_path(f"pseudo_cl_{ver}.sacc") - if os.path.exists(out_path): - self.print_done(f"Skipping Pseudo-Cl's calculation, {out_path} exists") - self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc(out_path) + ver_out_path = out_path or self._output_path(f"pseudo_cl_{ver}.sacc") + if os.path.exists(ver_out_path): + self.print_done( + f"Skipping Pseudo-Cl's calculation, {ver_out_path} exists" + ) + self._pseudo_cls[ver]["pseudo_cl"] = self._load_pseudo_cl_sacc( + ver_out_path + ) elif self.cell_method == "map": - self.calculate_pseudo_cl_map(ver, nside, out_path) + self.calculate_pseudo_cl_map(ver, nside, ver_out_path) elif self.cell_method == "catalog": - self.calculate_pseudo_cl_catalog(ver, out_path) + self.calculate_pseudo_cl_catalog(ver, ver_out_path) else: raise ValueError(f"Unknown cell method: {self.cell_method}") diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index b7ac16b8..637b10c2 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -593,3 +593,32 @@ def test_calculate_pseudo_cl_catalog_end_to_end(cv, tmp_path): params = get_params_rho_tau(cv.cc[ver], survey=ver) _, cl_prim, _ = cv.get_pseudo_cls_catalog(catalog=cat_gal, params=params) npt.assert_allclose(ee, cl_prim[0], rtol=RTOL_CAT, atol=ATOL_CAT) + + +def test_calculate_pseudo_cl_out_path_born_at_declared_name(cv): + """calculate_pseudo_cl(out_path=...) writes to the given path, not the + untagged native name — the anti-collision seam. + + The tagged producer (rule pseudo_cl, blind=A) and the untagged diagnostic + (rule cv_pseudo_cl, blind=None) both call calculate_pseudo_cl; if the tagged + one wrote the native pseudo_cl_{ver}.sacc and renamed, its skip-if-exists + could silently adopt — and the rename delete — the diagnostic's differently- + blinded file. Born-at-declared-name makes the two paths provably disjoint. + """ + ver = cv._test_version + cv._pseudo_cls = {} + tagged = cv._output_path(f"pseudo_cl_{ver}_blind=A_powspace_nbins=32.sacc") + native = cv._output_path(f"pseudo_cl_{ver}.sacc") + + cv.calculate_pseudo_cl(out_path=tagged) + + assert os.path.exists(tagged) + assert not os.path.exists(native) # no undeclared native basename touched + + +def test_calculate_pseudo_cl_out_path_rejects_multiversion(cv): + """out_path targets one part; a multi-version instance must fail loudly + rather than write every version to the same path.""" + cv.versions = [cv._test_version, "SecondVersion"] + with pytest.raises(ValueError, match="one part to one path"): + cv.calculate_pseudo_cl(out_path=cv._output_path("pseudo_cl_x.sacc")) diff --git a/workflow/scripts/generate_pseudo_cl.py b/workflow/scripts/generate_pseudo_cl.py index 81f69414..9bfabf4a 100644 --- a/workflow/scripts/generate_pseudo_cl.py +++ b/workflow/scripts/generate_pseudo_cl.py @@ -35,7 +35,7 @@ def generate_pseudo_cl( version: str, - output_dir: str, + out_path: str, cat_config: str, nside: int = 1024, npatch: int = 1, @@ -45,16 +45,16 @@ def generate_pseudo_cl( nbins: int = None, power: float = 0.5, ): - """Generate a pseudo-Cl data vector into ``output_dir``. + """Generate a pseudo-Cl data vector, born as a SACC part at ``out_path``. Parameters ---------- version : str Catalog version (e.g., "SP_v1.4.6_leak_corr") - output_dir : str - Directory the pseudo-Cl SACC part is written into. The primitive writes - its native ``pseudo_cl_{version}.sacc`` here; callers that need a tagged - filename rename it themselves (see ``_from_snakemake``). + out_path : str + Exact destination the SACC part is *born at* — its final (possibly + tagged) name. No native-basename + rename step, so this producer's + skip-if-exists never collides with the untagged cv_pseudo_cl diagnostic. cat_config : str Path to catalog configuration YAML nside : int @@ -76,8 +76,9 @@ def generate_pseudo_cl( Returns ------- str - Path to the primitive's native ``pseudo_cl_{version}.sacc`` product. + ``out_path`` (the SACC part written). """ + output_dir = os.path.dirname(out_path) os.makedirs(output_dir, exist_ok=True) blind_str = f" blind={blind}" if blind else "" @@ -136,25 +137,26 @@ def generate_pseudo_cl( cv = CosmologyValidation(**cv_kwargs) # Calculate pseudo-Cls only (no covariance). The data vector is born as a - # SACC part: pseudo_cl_{version}.sacc under output_dir. - cv.calculate_pseudo_cl() - - # Report on the native product (renamed by the Snakemake caller, if any). - src_cl = os.path.join(output_dir, f"pseudo_cl_{version}.sacc") - if os.path.exists(src_cl): - s = sacc_io.load(src_cl) + # SACC part directly at out_path (its final, possibly-tagged name) — no + # shared native basename, no rename, so this producer's skip-if-exists never + # collides with the untagged cv_pseudo_cl diagnostic (which would otherwise + # let one rule adopt + delete the other's differently-blinded file). + cv.calculate_pseudo_cl(out_path=out_path) + + if os.path.exists(out_path): + s = sacc_io.load(out_path) ell = sacc_io.get_pseudo_cl(s, (0, 0))[0] print(f"Generated pseudo-Cl with {len(ell)} ell bins") print(f"ell range: [{ell.min():.1f}, {ell.max():.1f}]") - return src_cl + return out_path def _from_snakemake(smk): p = smk.params - output_cl = smk.output.pseudo_cl - src_cl = generate_pseudo_cl( + # Born directly at the rule's declared (tagged) output — no rename step. + generate_pseudo_cl( version=p["version"], - output_dir=os.path.dirname(output_cl), + out_path=smk.output.pseudo_cl, cat_config=p["cat_config"], nside=int(p["nside"]), npatch=int(p["npatch"]), @@ -164,10 +166,6 @@ def _from_snakemake(smk): nbins=int(p["nbins"]), power=float(p.get("power", 0.5)), ) - # Snakemake declares a tagged output filename; rename the native product to it. - if os.path.exists(src_cl) and src_cl != output_cl: - os.rename(src_cl, output_cl) - print(f"Saved to: {output_cl}") def _from_cli(argv=None): @@ -220,9 +218,13 @@ def _from_cli(argv=None): with open(a.cosmo_json) as f: cosmo_params = json.load(f) + # lc/ASTRA path: --out is a per-recipe directory; the untagged native name + # is unambiguous there (each recipe gets its own tree, so no cross-nbins or + # cross-blind collision). + out_path = os.path.join(a.out, f"pseudo_cl_{a.ver}.sacc") generate_pseudo_cl( version=a.ver, - output_dir=a.out, + out_path=out_path, cat_config=a.cat_config, nside=a.nside, npatch=a.npatch, From e0519c85199e9cc6df41cdf59330cc8d582f23f9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:20:17 +0200 Subject: [PATCH 23/38] =?UTF-8?q?fix(workflow):=20assemble=20fails=20loudl?= =?UTF-8?q?y=20by=20default=20on=20the=20placeholder=20=CE=BE=20covariance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit {version}.sacc is the terminal science file; shipping it with a var=1.0 diagonal as the LEADING (ξ±) covariance block — ~20 orders off the real variance — is a silent catastrophic χ²/PTE for any consumer. Drop the unconditional placeholder_var=1.0. Default: no real ξ-cov wired → assemble_sacc.py's existing ValueError fires. The placeholder is now gated behind an explicit opt-in (cosmo_val.allow_placeholder_cov: true), for dry-run / test configs only. The pseudo-Cℓ block stays real (pseudo_cl_cov input); the integration tests pass placeholder_var explicitly and are unaffected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- workflow/rules/cosmo_val.smk | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index 3fe0e1df..d9a56403 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -490,11 +490,16 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", - # ξ± coarse block: documented diagonal placeholder until PR-3's converter - # sources the real CosmoCov theory covariance via --xi-cov. The pseudo-Cℓ - # block is real (pseudo_cl_cov input); COSEBIs / pure-E/B / ρ/τ carry - # their own. assemble_sacc.py reads pseudo_cl_cov's COVAR_* extensions. - placeholder_var=1.0, + # ξ± coarse has no real covariance wired yet (its CosmoCov theory block is + # PR-3's converter territory, plugging in via --xi-cov). By DEFAULT this + # is fatal: assemble_sacc.py raises rather than ship {version}.sacc — the + # terminal science file — with a var=1.0 placeholder as its LEADING + # covariance block (~20 orders off the real ξ± variance → silent + # catastrophic χ²/PTE for any consumer). Only an explicit config opt-in + # (cosmo_val.allow_placeholder_cov: true — dry-run / test configs) attaches + # the flagged diagonal placeholder. The pseudo-Cℓ block is real (from the + # pseudo_cl_cov input); COSEBIs / pure-E/B / ρ/τ carry their own. + placeholder_var=(1.0 if CV.get("allow_placeholder_cov", False) else None), resources: mem_mb=8000, runtime=20, From 6a5df5b701675bf66b987cc041264073cbf228b9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:22:23 +0200 Subject: [PATCH 24/38] test(workflow): dry-run guard covers the cosmo_val assemble DAG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_bmodes_workflow_dry_runs uses papers/bmodes, whose config has no cosmo_val block, so cosmo_val.smk (the born-as-SACC + assemble rules) is never included — the guard couldn't catch a break in them. Add test_cosmo_val_workflow_assemble_ dry_runs targeting assemble_sacc_all in papers/cosmo_val (the only paper that includes cosmo_val.smk): asserts the DAG resolves and each assemble_sacc job pulls the tagged, blinded pseudo-Cl part + its NaMaster cov (not the untagged diagnostic) plus all five per-statistic parts. Shared _dry_run helper factored out of the bmodes guard. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- .../tests/test_bmodes_workflow_dry_run.py | 70 ++++++++++++++----- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 60675f52..87afc416 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -1,7 +1,10 @@ -"""Back-pressure guard #2: the B-modes Snakemake workflow dry-runs. +"""Back-pressure guard #2: the paper Snakemake workflows dry-run. -The reorg is allowed to change the rule graph; this guard only asserts that -Snakemake can still parse the workflow and construct a dry run. +The reorg is allowed to change the rule graph; these guards only assert that +Snakemake can still parse each composed workflow and construct a dry run. One +guard covers papers/bmodes (config space, no cosmo_val block); a second covers +papers/cosmo_val, whose config DOES carry a cosmo_val block — so it is the only +one that includes cosmo_val.smk and hence the born-as-SACC + assemble rules. """ import os @@ -27,38 +30,69 @@ def _repo_root() -> Path: raise RuntimeError("could not locate repo root (no pyproject.toml above test)") -@requires_candide_data -def test_bmodes_workflow_dry_runs(): - """The paper B-mode workflow must still parse and dry-run cleanly.""" - workflow_dir = _repo_root() / "papers/bmodes" - # PYTHONUNBUFFERED satisfies the Snakefile's `envvars:` declaration without - # depending on the invoking shell's environment. A dry run resolves the DAG - # only — it never dispatches jobs — so drop any inherited SNAKEMAKE_PROFILE - # (e.g. the login shell's "slurm" profile), which would otherwise force an - # executor plugin the test environment need not have installed. +def _dry_run(workflow_dir, targets, *extra_snakemake_args): + """Construct a dry run of the paper workflow at ``workflow_dir``. + + Returns the CompletedProcess. PYTHONUNBUFFERED satisfies the Snakefile's + ``envvars:`` declaration without depending on the invoking shell. A dry run + resolves the DAG only — it never dispatches jobs — so drop any inherited + SNAKEMAKE_PROFILE (e.g. the login shell's "slurm" profile), which would + otherwise force an executor plugin the test environment need not have. And + invoke snakemake through sys.executable (the interpreter pytest, hence + snakemake, lives in) — a bare python3.12 resolves off PATH to e.g. an + intel-python without snakemake. + """ env = os.environ | {"PYTHONNOUSERSITE": "1", "PYTHONUNBUFFERED": "1"} env.pop("SNAKEMAKE_PROFILE", None) - result = subprocess.run( + return subprocess.run( [ - # Invoke snakemake through the interpreter running the test — a bare - # "python3.12" resolves off PATH (e.g. intel-python without snakemake); - # sys.executable is the environment that pytest, hence snakemake, lives in. sys.executable, "-m", "snakemake", - "all_tapestry", + *targets, "--dry-run", "--cores", "1", "--configfile", "config/config.yaml", + *extra_snakemake_args, ], cwd=workflow_dir, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - timeout=60, + timeout=120, check=False, ) + + +@requires_candide_data +def test_bmodes_workflow_dry_runs(): + """The paper B-mode workflow must still parse and dry-run cleanly.""" + result = _dry_run(_repo_root() / "papers/bmodes", ["all_tapestry"]) + assert result.returncode == 0, result.stdout + + +@requires_candide_data +def test_cosmo_val_workflow_assemble_dry_runs(): + """The cosmo_val workflow (the only one including cosmo_val.smk) resolves the + born-as-SACC + assemble DAG, and assemble pulls the tagged pseudo-Cl + cov. + + Targets the assemble_sacc_all rule so every version's assemble_sacc job + appears. The dry run resolves the DAG structure only — it never executes the + assemble script — so the placeholder-cov opt-in (cosmo_val.allow_placeholder_cov) + is irrelevant here; a real run would need it (or a wired --xi-cov) to proceed, + which is the fail-loud-by-default behaviour asserted in test_assemble_sacc.""" + version = "SP_v1.4.6.3_leak_corr" + result = _dry_run(_repo_root() / "papers/cosmo_val", ["assemble_sacc_all"]) assert result.returncode == 0, result.stdout + # assemble_sacc must be in the DAG and pull the tagged, blinded pseudo-Cl + # part + its NaMaster covariance (not the untagged cv_pseudo_cl diagnostic), + # plus all five per-statistic parts. + out = result.stdout + assert "rule assemble_sacc:" in out, out + assert f"pseudo_cl_{version}_blind=A_powspace_nbins=32.sacc" in out, out + assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out + for part in ("_xi_coarse_", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): + assert part in out, f"missing {part} part in assemble DAG:\n{out}" From 3561b3692fd0444e40959055d5a70fc354615c9c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:24:03 +0200 Subject: [PATCH 25/38] fix(bmodes): drop removed save_fits kwarg from run_xi_sweep's run_2pcf call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_xi_sweep.py called run_2pcf(..., save_fits=True) — a kwarg the SACC migration removed, so every invocation TypeErrors. The sweep consumes only the .txt dump; drop the kwarg. run_2pcf is born-as-SACC, so give its coarse part a grid-qualified sacc_out (the default {ver}_xi_coarse.sacc carries no binning, so the reporting + integration grids would collide per version). Stale "+ ξ+/ξ- FITS" docstring dropped. test_cli_seams: bind the sweep's exact run_2pcf call against the live signature (and assert save_fits no longer binds) so this CLI seam can't silently rot — the compute is cluster-only and never exercised by the fast suite. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- papers/bmodes/scripts/run_xi_sweep.py | 10 ++-- src/sp_validation/tests/test_cli_seams.py | 65 +++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 src/sp_validation/tests/test_cli_seams.py diff --git a/papers/bmodes/scripts/run_xi_sweep.py b/papers/bmodes/scripts/run_xi_sweep.py index b02283a5..49076435 100644 --- a/papers/bmodes/scripts/run_xi_sweep.py +++ b/papers/bmodes/scripts/run_xi_sweep.py @@ -2,8 +2,8 @@ Loops the [non-fiducial version list](sweep_versions.nonfiducial_versions) and runs the same ``run_2pcf.run_2pcf`` compute the fiducial two_point recipes call, -once per version, writing every version's ξ± text dump (+ ξ+/ξ- FITS) into one -lc ``{output}`` dir under run_2pcf's native, already-canonical name +once per version, writing every version's ξ± text dump into one lc ``{output}`` +dir under run_2pcf's native, already-canonical name ``{ver}_xi_minsep={min}_maxsep={max}_nbins={nbins}_npatch={npatch}.txt`` — the exact pattern ``cosebis_version_comparison._xi_integration`` reconstructs. @@ -72,11 +72,15 @@ def _from_cli(argv=None): versions = a.versions or nonfiducial_versions(config) for ver in versions: for grid in a.grids: + # The sweep consumes only the .txt dump (cosebis_version_comparison + # reconstructs it by binning). run_2pcf is born-as-SACC, so give its + # coarse part a grid-qualified name — the default {ver}_xi_coarse.sacc + # carries no binning, so the two grids per version would collide. run_2pcf( ver=ver, cat_config=a.cat_config, output_dir=a.out, - save_fits=True, + sacc_out=os.path.join(a.out, f"{ver}_xi_coarse_{grid}.sacc"), **GRIDS[grid], ) diff --git a/src/sp_validation/tests/test_cli_seams.py b/src/sp_validation/tests/test_cli_seams.py new file mode 100644 index 00000000..697b3426 --- /dev/null +++ b/src/sp_validation/tests/test_cli_seams.py @@ -0,0 +1,65 @@ +"""Smoke tests for workflow CLI seams — cheap guards against signature rot. + +A CLI script that calls a workflow function with a removed/renamed kwarg +TypeErrors only at invocation time (the compute is cluster-only, so it is never +exercised by the fast suite). These tests bind the exact call each seam makes +against the current signature via ``inspect.signature(...).bind(...)`` — no +compute, no data — so a drifted kwarg (e.g. run_xi_sweep's dropped save_fits) +fails here instead of on the cluster. +""" + +import importlib.util +import inspect +from pathlib import Path + +import pytest + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / "pyproject.toml").exists(): + return parent + raise RuntimeError("could not locate repo root (no pyproject.toml above test)") + + +def _load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_run_xi_sweep_run_2pcf_call_binds(): + """The kwargs run_xi_sweep passes to run_2pcf must bind to its signature. + + Mirrors the call in papers/bmodes/scripts/run_xi_sweep.py — if run_2pcf drops + or renames a parameter (save_fits was removed by the SACC migration), the + bind raises TypeError here rather than on every cluster invocation. + """ + root = _repo_root() + run_2pcf_mod = _load(root / "workflow/scripts/run_2pcf.py", "run_2pcf_seam") + sig = inspect.signature(run_2pcf_mod.run_2pcf) + # Exactly the keyword set run_xi_sweep._from_cli passes (grid params spread + # from GRIDS: min_sep/max_sep/nbins/npatch). + sig.bind( + ver="V", + cat_config="/cfg.yaml", + output_dir="/out", + sacc_out="/out/V_xi_coarse_reporting.sacc", + min_sep=1.0, + max_sep=250.0, + nbins=20, + npatch=1, + ) + # And the removed kwarg must NOT bind (guards against a silent re-add). + with pytest.raises(TypeError): + sig.bind( + ver="V", + cat_config="/cfg.yaml", + output_dir="/out", + save_fits=True, + min_sep=1.0, + max_sep=250.0, + nbins=20, + npatch=1, + ) From 149a50c523cc1ccfeb78f99809ecf2f9f6ccc9da Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 12:26:01 +0200 Subject: [PATCH 26/38] fix(assemble_sacc): validate expected parts, no silent statistic drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _from_snakemake built part_paths with a hasattr filter, so a typo'd input keyword (cosebi for cosebis) silently omitted that statistic from the terminal {version}.sacc — the exact silent-truncation class this series has been bitten by. assemble_sacc now takes an `expected` list (the statistics the caller wired, from its config toggles) and raises if any is missing from part_paths or names a non-CANONICAL statistic. The rule derives expected from cv_assemble_inputs so it tracks the include_pseudo_cl toggle. CLI path is already typo-safe (argparse rejects unknown flags), so it passes expected=None. Tests: typo'd input key raises; typo in the expected list itself raises. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HPfbe2XVTGzbnPn4g7BdN6 --- src/sp_validation/tests/test_assemble_sacc.py | 27 +++++++++++++++++++ workflow/rules/cosmo_val.smk | 6 +++++ workflow/scripts/assemble_sacc.py | 26 ++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index a43747dc..ccddab97 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -240,3 +240,30 @@ def test_assemble_sacc_respects_pseudo_cl_toggle(tmp_path): s2 = sio.load(str(out)) assert type(s2.covariance).__name__ == "FullCovariance" assert s2.covariance.dense.shape == (len(s2.mean), len(s2.mean)) + + +def test_assemble_sacc_expected_part_missing_raises(tmp_path): + """A typo'd input keyword drops a part from part_paths; the expected list + catches it rather than silently omitting the statistic.""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + # Simulate a rule-input typo: cosebis wired under the wrong key. + paths["cosebi"] = paths.pop("cosebis") + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="expected parts \\['cosebis'\\] missing"): + asm.assemble_sacc( + "vSYNTH", + paths, + str(out), + expected=["xi_coarse", "pseudo_cl", "cosebis", "pure_eb", "rho_tau"], + placeholder_var=1.0, + ) + + +def test_assemble_sacc_expected_rejects_unknown_name(tmp_path): + """A typo in the expected list itself is rejected (not a valid statistic).""" + paths = _write_parts(tmp_path, cov_less=("xi_coarse",)) + out = tmp_path / "vSYNTH.sacc" + with pytest.raises(ValueError, match="not assemblable statistics"): + asm.assemble_sacc( + "vSYNTH", paths, str(out), expected=["cosebi"], placeholder_var=1.0 + ) diff --git a/workflow/rules/cosmo_val.smk b/workflow/rules/cosmo_val.smk index d9a56403..4f2a2e46 100644 --- a/workflow/rules/cosmo_val.smk +++ b/workflow/rules/cosmo_val.smk @@ -490,6 +490,12 @@ rule assemble_sacc: sacc=cv_analysis_sacc("{version}"), params: version="{version}", + # Statistics this rule wired (same toggles as cv_assemble_inputs). The + # script validates part_paths against this so a typo'd input keyword + # can't silently drop a statistic from the terminal file. + expected=lambda w: [ + k for k in cv_assemble_inputs(w.version) if k != "pseudo_cl_cov" + ], # ξ± coarse has no real covariance wired yet (its CosmoCov theory block is # PR-3's converter territory, plugging in via --xi-cov). By DEFAULT this # is fatal: assemble_sacc.py raises rather than ship {version}.sacc — the diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 2ad6fa27..513ec976 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -107,6 +107,7 @@ def assemble_sacc( part_paths, out_path, *, + expected=None, xi_cov=None, pseudo_cl_cov=None, pseudo_cl_cov_hdu="COVAR_FULL", @@ -123,9 +124,29 @@ def assemble_sacc( present statistics are assembled; order is forced to canonical. out_path : str Destination ``{version}.sacc``. + expected : sequence of str, optional + Statistics that MUST be present in ``part_paths`` (from the caller's + config toggles). Raises loudly if any is missing or has no path — so a + typo'd input keyword (``cosebi`` for ``cosebis``) can't silently drop a + statistic from the terminal file. Names not in :data:`CANONICAL` are + rejected too (catches a typo in the expected list itself). xi_cov, pseudo_cl_cov, pseudo_cl_cov_hdu, placeholder_var Covariance sourcing — see the module docstring. """ + if expected is not None: + unknown = [name for name in expected if name not in CANONICAL] + if unknown: + raise ValueError( + f"expected parts {unknown} are not assemblable statistics; " + f"valid names are {CANONICAL}" + ) + missing = [name for name in expected if not part_paths.get(name)] + if missing: + raise ValueError( + f"expected parts {missing} missing from part_paths for {version} " + f"(got {sorted(part_paths)}); a required statistic would be " + "silently dropped from the terminal analysis file" + ) parts = [] nz = metadata = None for name in CANONICAL: @@ -167,10 +188,15 @@ def _from_snakemake(smk): for name in CANONICAL if hasattr(inp, name) and getattr(inp, name) } + # The rule declares which statistics it wired (from its config toggles); a + # typo in an input keyword drops the part from part_paths above, so validate + # against this expected list rather than trusting the hasattr filter. + expected = list(p["expected"]) assemble_sacc( version=p["version"], part_paths=part_paths, out_path=str(smk.output[0]), + expected=expected, xi_cov=getattr(inp, "xi_cov", None), pseudo_cl_cov=getattr(inp, "pseudo_cl_cov", None), pseudo_cl_cov_hdu=p.get("pseudo_cl_cov_hdu", "COVAR_FULL"), From df87be6d7ff47fb4904fa6709c4cc9aa67b746bf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 16:34:47 +0200 Subject: [PATCH 27/38] feat(sacc_like): sp_validation shim over CosmoSIS SaccClLikelihood MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopt CosmoSIS's native SACC likelihood for the ξ± inference path through a thin sp_validation-owned subclass that fixes two upstream defects: 1. arcmin→rad theta-tag conversion — upstream sacc_like builds its theory spline on block[section,"theta"] (radians) but evaluates it at raw SACC theta tags (arcmin), silently returning theory≈0 and collapsing χ². 2pt_like converts to radians for exactly this reason; sacc_like never does. 2. an ordering guard — the data vector (get_mean, insertion order) and the theory loop (get_data_types × tracer_combinations × points) agree only for type-major files; a pair-major tomographic file would silently misalign. The guard reconstructs the theory-loop order and requires it == arange. Factory pattern (setup imports the upstream class from csl_dir and subclasses at call time) so importing the module never needs cosmosis; not imported by __init__. build_data calls super() first (scale cuts run in arcmin, matching 2pt_like's angle_range grammar) then converts + guards. Tests (cosmosis + CSL_DIR gated, skip cleanly otherwise) prove in-process equality against 2pt_like on the PR-3 converter FITS: machine-zero equality on synthetic (χ²=1.18829133107, Δχ²=0) and real data (χ²=420310.753739, Δχ²=0, N=40), the unit-gap tripwire (raw sacc_like 631× off), scale-cut equivalence, identical perturbation response, the ordering guard, and real-type-scoped theta conversion. pyproject: cosmosis>=3.25 extra (inert in CI, which installs [test,glass,blinding]). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- pyproject.toml | 9 + src/sp_validation/sacc_like_unions.py | 186 ++++++++ src/sp_validation/tests/test_sacc_like.py | 506 ++++++++++++++++++++++ 3 files changed, 701 insertions(+) create mode 100644 src/sp_validation/sacc_like_unions.py create mode 100644 src/sp_validation/tests/test_sacc_like.py diff --git a/pyproject.toml b/pyproject.toml index ecdec6f1..eef183fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,6 +138,15 @@ blinding = [ # sp_validation fast suite. "numpy>=2.2,<2.5", ] +# Native SACC inference likelihood (PR 7, sp_validation.sacc_like_unions). The +# shim subclasses CosmoSIS's SaccClLikelihood; this pins the engine. Both engine +# module files are pure-python (no CAMB / compiled CSL), so the equality tests +# (tests/test_sacc_like.py) run with just this extra + a CosmoSIS Standard +# Library checkout, located via the CSL_DIR env var: +# git clone --depth 1 https://github.com/joezuntz/cosmosis-standard-library CSL +# CSL_DIR=/path/to/CSL pytest ... test_sacc_like.py +# Inert in CI (its Dockerfile installs only [test,glass,blinding]). +cosmosis = ["cosmosis>=3.25"] develop = ["sp_validation[test,docs]"] [tool.pytest.ini_options] diff --git a/src/sp_validation/sacc_like_unions.py b/src/sp_validation/sacc_like_unions.py new file mode 100644 index 00000000..2b7e5cc6 --- /dev/null +++ b/src/sp_validation/sacc_like_unions.py @@ -0,0 +1,186 @@ +"""SACC likelihood shim for CosmoSIS — the sp_validation-owned native path. + +This module is a thin subclass of CosmoSIS's ``SaccClLikelihood`` (from the +CosmoSIS Standard Library, ``likelihood/sacc/sacc_like.py``) that fixes two +upstream defects so the native SACC likelihood matches the PR-3 converter → +``2pt_like`` path bit for bit on our real-space ξ± analysis file. It exists as a +CosmoSIS *module file* (``setup``/``execute``/``cleanup`` at module scope), loaded +via ``file = .../sacc_like_unions.py`` in an ini, and is NOT imported by +``sp_validation/__init__`` (cosmosis is an optional dependency). + +Why the shim exists +------------------- +Two things in upstream ``sacc_like`` break a real-space ξ likelihood; both are +documented and empirically verified (probe: Δχ²=3184 raw vs Δχ²=0 shimmed on the +single-bin analysis file), and both are fixed here by overriding ``build_data``: + +1. **No arcmin→radian conversion (the killer).** + ``sacc_likelihoods/twopoint.py`` (L74-90 at CSL commit 4fd2f1c) builds a + ``SpectrumInterp`` over ``block[section, "theta"]`` — theory θ in *radians* + (CosmoSIS convention) — and evaluates it at each data point's raw ``theta`` + tag, which our SACC files (and firecrown) store in *arcmin*. ``2pt_like.py`` + (L179-183) converts its real-space data to radians for exactly this reason; + ``sacc_like`` never does, so the spline is evaluated ~3437× outside its grid, + ``SpectrumInterp`` returns 0 there, and χ² silently collapses to dᵀC⁻¹d. The + only prior use was the ℓ-space (unit-free) Cℓ path, which is why it was never + caught. We convert the ``theta`` tags to radians here, after scale cuts. + +2. **Theory↔data ordering is assumed, never enforced.** + The data vector is ``sacc.get_mean()`` (insertion order); the theory vector is + built by looping ``get_data_types() × get_tracer_combinations() × points``. + A comment in ``twopoint.py`` (L35) claims ``to_canonical_order`` was called on + load — it is not. The two orders agree only when the file is grouped + type-major (all of one data type, then the next). Our single-pair + ``[ξ+; ξ−]`` files satisfy this; a tomographic *pair-major* file (ξ+/ξ− per + pair) would silently misalign theory against data. We reconstruct the + theory-loop index order and require it equals ``arange`` — a hard ValueError + otherwise (see ``test_ordering_guard_raises_pair_major_tomographic``). + +When upstream fixes the units, this shim dies: the tripwire test +``test_upstream_unit_gap_tripwire`` in ``tests/test_sacc_like.py`` fails the day +raw ``sacc_like`` stops producing a wildly different χ², signalling the shim can +be retired. + +Requires a CSL checkout: ``setup`` reads ``csl_dir`` from the module options and +imports the upstream ``sacc_like`` from ``/likelihood/sacc``. The tests +locate it via the ``CSL_DIR`` environment variable (see the test module docstring +for the checkout recipe). +""" + +import os +import sys + +import numpy as np +from cosmosis.datablock import SectionOptions, option_section + +# arcmin → radian: the conversion 2pt_like applies to real-space data and that +# sacc_like omits. Applied only to `theta` tags of `real`-category data types. +ARCMIN_TO_RAD = np.pi / (180.0 * 60.0) + + +def _import_upstream_sacc_like(csl_dir): + """Import the upstream ``sacc_like`` module from a CSL checkout. + + ``/likelihood/sacc`` is prepended to ``sys.path`` so both + ``sacc_like`` and its sibling ``sacc_likelihoods`` package (imported by + ``sacc_like`` for the theory-extraction functions) resolve. Idempotent: the + path is only inserted once. + """ + sacc_dir = os.path.join(csl_dir, "likelihood", "sacc") + if not os.path.isdir(sacc_dir): + raise ValueError( + f"csl_dir={csl_dir!r} has no likelihood/sacc directory; point csl_dir " + "at a CosmoSIS Standard Library checkout (see module docstring)" + ) + if sacc_dir not in sys.path: + sys.path.insert(0, sacc_dir) + import sacc_like # noqa: E402 — resolved from the sys.path insertion above + + return sacc_like + + +def _make_subclass(sacc_like): + """Build ``SaccLikeUnions`` as a subclass of the upstream ``SaccClLikelihood``. + + A factory (not an import-time ``class ... :`` statement) so importing this + module never requires the upstream class — that dependency is deferred to + ``setup``, which has ``csl_dir`` in hand. Only ``build_data`` is overridden; + scale cuts, covariance handling (Sellentin/Hartlap), theory extraction and + ``save_theory`` all ride upstream unmodified. + """ + + class SaccLikeUnions(sacc_like.SaccClLikelihood): + """CSL ``SaccClLikelihood`` with the arcmin→rad + ordering-guard fixes.""" + + def build_data(self): + # Run the upstream build FIRST: it loads the SACC, applies data_sets + # selection and the arcmin-grammar scale cuts (matching 2pt_like's + # angle_range convention and the ini ergonomics), populates + # self.sacc_data / self.sections_for_names, and returns the + # (unit-independent) data vector we pass straight through. + x, data_vector = super().build_data() + + self._convert_real_theta_tags_to_radians() + self._assert_theory_order_matches_data() + + return x, data_vector + + def _convert_real_theta_tags_to_radians(self): + """Scale the ``theta`` tag arcmin→rad for every ``real``-category point. + + The theory spline is built on ``block[section, "theta"]`` in radians, + so the ``theta`` tag each point is evaluated at must be radians too. + Scoped to data types whose category (``sections_for_names[dt][0]``) is + ``real``: cosebis ``n`` tags and spectrum ``ell`` tags are unit-free + and must not be touched, and only the ``theta`` tag is converted + (``theta_nom`` etc. are metadata the likelihood never evaluates). + """ + real_types = { + dt + for dt, (category, _section) in self.sections_for_names.items() + if category == "real" + } + for point in self.sacc_data.data: + if point.data_type in real_types and "theta" in point.tags: + point.tags["theta"] = point.tags["theta"] * ARCMIN_TO_RAD + + def _assert_theory_order_matches_data(self): + """Require the theory-loop order to equal the data-vector order. + + The data vector is ``sacc.get_mean()`` (insertion order); upstream + builds theory by looping data types, then tracer combinations, then + points, and concatenating — assuming (never enforcing) that this + reproduces insertion order. It does only for type-major files. We + reconstruct that loop's index order and require it be ``arange``; + otherwise theory and data would be silently misaligned (the same bug + class the PR-2/PR-3 reviews caught for the converter). + """ + order = [ + int(i) + for dt in self.sacc_data.get_data_types() + for tracers in self.sacc_data.get_tracer_combinations(dt) + for i in self.sacc_data.indices(dt, tracers) + ] + expected = np.arange(len(self.sacc_data.mean)) + if not np.array_equal(order, expected): + raise ValueError( + "SACC data/theory ordering mismatch: the theory loop " + "(get_data_types × get_tracer_combinations × points) does not " + "reproduce the get_mean() insertion order, so sacc_like would " + "compare theory against data point-by-point in the WRONG order " + "and return a silently wrong χ². This happens when the file is " + "grouped pair-major (ξ+/ξ− interleaved per tracer pair) rather " + "than type-major (all ξ+, then all ξ−). Upstream assumes " + "to_canonical_order() was applied on load but never calls it; " + "write the SACC type-major, or call to_canonical_order() before " + "saving." + ) + + return SaccLikeUnions + + +def setup(options): + """CosmoSIS ``setup`` — build and instantiate the shimmed likelihood. + + Mirrors ``GaussianLikelihood.build_module``'s setup: wrap the raw options in + ``SectionOptions`` and instantiate the likelihood (whose ``__init__`` calls + ``build_data``). The one addition is reading ``csl_dir`` from the module + options to locate and import the upstream class before subclassing it. + """ + csl_dir = options.get_string(option_section, "csl_dir") + sacc_like = _import_upstream_sacc_like(csl_dir) + likelihood_class = _make_subclass(sacc_like) + return likelihood_class(SectionOptions(options)) + + +def execute(block, config): + """CosmoSIS ``execute`` — run the likelihood (mirrors ``build_module``).""" + likelihood_calculator = config + likelihood_calculator.do_likelihood(block) + return 0 + + +def cleanup(config): + """CosmoSIS ``cleanup`` — mirror of ``build_module``'s cleanup.""" + likelihood_calculator = config + likelihood_calculator.cleanup() diff --git a/src/sp_validation/tests/test_sacc_like.py b/src/sp_validation/tests/test_sacc_like.py new file mode 100644 index 00000000..f8c6c3b2 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_like.py @@ -0,0 +1,506 @@ +"""Equality tests: the sp_validation SACC-likelihood shim vs CosmoSIS ``2pt_like``. + +PR 7 adopts CosmoSIS's native ``SaccClLikelihood`` for the ξ± inference path, +through the shim :mod:`sp_validation.sacc_like_unions` (which fixes the upstream +arcmin→rad gap and adds an ordering guard). The contract is *in-process module +equality*: run the shimmed ``sacc_like`` on the analysis SACC and CosmoSIS's +``2pt_like`` on the PR-3 converter's 2pt-FITS against an identical synthetic +theory DataBlock, and require the same χ², log-likelihood, theory vector and +post-cut point count. The prototype (``sacc-like-probe/probe_equality.py``) +observed *exact* equality (Δχ²=0, Δtheory=0), so the equality tests assert +``array_equal`` / rtol=1e-12. + +Environment +----------- +Both engines are pure-python CosmoSIS module files (no CAMB, no compiled CSL +modules), so they run in the shared venv with ``cosmosis`` installed. They need a +checkout of the CosmoSIS Standard Library, located via the ``CSL_DIR`` env var; +the module is skipped when cosmosis is absent (CI image) or ``CSL_DIR`` is unset +or missing. Recipe:: + + git clone --depth 1 https://github.com/joezuntz/cosmosis-standard-library CSL + CSL_DIR=/path/to/CSL pytest ... test_sacc_like.py + +The pyproject ``cosmosis`` extra pins ``cosmosis>=3.25`` for the engine itself +(inert in CI, whose Dockerfile installs only ``[test,glass,blinding]``). +""" + +import importlib.util +import os +from pathlib import Path + +import numpy as np +import pytest + +cosmosis = pytest.importorskip("cosmosis") + +# The upstream CSL checkout carrying likelihood/sacc + likelihood/2pt. Skip the +# whole module (not error) when it is not configured, mirroring the cosmo_numba +# env-gated precedent. +_CSL_DIR = os.environ.get("CSL_DIR") +if not _CSL_DIR or not Path(_CSL_DIR, "likelihood", "sacc").is_dir(): + pytest.skip( + "CSL_DIR unset or has no likelihood/sacc — set CSL_DIR to a CosmoSIS " + "Standard Library checkout to run the sacc_like equality tests", + allow_module_level=True, + ) + +from cosmosis.datablock import DataBlock, option_section # noqa: E402 + +from sp_validation import sacc_io, twopoint_convert # noqa: E402 + +# Reuse the angular-bin count from the converter tests so the two suites' single- +# bin shapes stay in lockstep. (The χ²-dynamics builders here need a covariance +# commensurate with the data, which the converter's byte-compare _sacc is not.) +from sp_validation.tests.test_twopoint_convert import N_ANG # noqa: E402 + +CSL = Path(_CSL_DIR) +ARCMIN_TO_RAD = np.pi / (180.0 * 60.0) + +# The like_name both engines are configured with, so both write identical block +# keys (_CHI2, _LIKE, _theory) — the parity the design mandates. +LIKE_NAME = "2pt_like" + + +# --------------------------------------------------------------------------- +# Shared engine harness +# --------------------------------------------------------------------------- +def _load_module_file(path, name): + """Import a CosmoSIS module file (``setup``/``execute``/``cleanup``) by path. + + ``likelihood/sacc`` and ``likelihood/2pt`` are added to ``sys.path`` so the + upstream modules resolve their siblings (``sacc_likelihoods``, ``spec_tools``, + ``twopoint_cosmosis``, …). Loading ``2pt_like.py`` runs its + ``build_module()`` at import (module-level ``setup, execute, cleanup``); + ``sacc_like_unions.py`` defines those functions directly. + """ + import sys + + for sub in ("likelihood/sacc", "likelihood/2pt"): + p = str(CSL / sub) + if p not in sys.path: + sys.path.insert(0, p) + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def _theory_block(): + """A DataBlock carrying the synthetic ξ± theory both engines interpolate. + + Mirrors the probe: the theory θ grid is in *radians* (CosmoSIS convention), + and the ξ+/ξ− predictions are smooth power laws sampled on it. Both engines + build a spline over this grid and evaluate it at each data point's angular + tag — so as long as both see the same block, the interpolated theory (and + hence χ²) must agree. + """ + theta_arcmin = np.geomspace(0.3, 400.0, 300) + + def t_xip(th): + return 2e-4 * (th / 10.0) ** -0.8 + + def t_xim(th): + return 1e-4 * (th / 10.0) ** -0.5 + + b = DataBlock() + for section, f in (("shear_xi_plus", t_xip), ("shear_xi_minus", t_xim)): + b[section, "theta"] = theta_arcmin * ARCMIN_TO_RAD + b[section, "bin_1_1"] = f(theta_arcmin) + b[section, "is_auto"] = True + b[section, "nbin_a"] = 1 + b[section, "nbin_b"] = 1 + b[section, "sample_a"] = "nz_source" + b[section, "sample_b"] = "nz_source" + b[section, "sep_name"] = "theta" + b[section, "save_name"] = "" + return b + + +def _run(mod, options): + """Run a CosmoSIS likelihood module file standalone; return (like, chi2, theory, n). + + Builds the options DataBlock (module options live under ``option_section``), + calls ``setup`` → ``execute`` on a fresh theory block, and reads the standard + Gaussian-likelihood outputs back out under the shared ``LIKE_NAME`` keys. + """ + opt = DataBlock() + for key, value in options.items(): + opt[option_section, key] = value + config = mod.setup(opt) + block = _theory_block() + mod.execute(block, config) + like = block["likelihoods", f"{LIKE_NAME}_LIKE"] + chi2 = block["data_vector", f"{LIKE_NAME}_CHI2"] + theory = block["data_vector", f"{LIKE_NAME}_theory"] + return like, chi2, np.asarray(theory), len(theory) + + +# --------------------------------------------------------------------------- +# The two engine module files + their options +# --------------------------------------------------------------------------- +_SHIM_PATH = Path(__file__).resolve().parents[1] / "sacc_like_unions.py" + + +@pytest.fixture(scope="module") +def m_shim(): + return _load_module_file(_SHIM_PATH, "sacc_like_unions_test") + + +@pytest.fixture(scope="module") +def m_2pt(): + return _load_module_file(CSL / "likelihood/2pt/2pt_like.py", "twopt_like_test") + + +@pytest.fixture(scope="module") +def m_raw_sacc(): + return _load_module_file(CSL / "likelihood/sacc/sacc_like.py", "raw_sacc_like_test") + + +def _shim_opts(sacc_path, **extra): + return { + "csl_dir": str(CSL), + "data_file": sacc_path, + "data_sets": "galaxy_shear_xi_plus galaxy_shear_xi_minus", + "like_name": LIKE_NAME, + **extra, + } + + +def _twopt_opts(fits_path, **extra): + return { + "data_file": fits_path, + "data_sets": "XI_PLUS XI_MINUS", + "covmat_name": "COVMAT", + "like_name": LIKE_NAME, + "gaussian_covariance": False, + "cut_zeros": False, + **extra, + } + + +def _realistic_sacc(seed=0, *, xip=None): + """A single-bin ξ± SACC with data + covariance sized like the real product. + + The converter-test ``_sacc`` builder uses a covariance ~14 orders of + magnitude larger than the ξ values (fine for byte-comparing the converter, + where covariance *content* is irrelevant), which makes every χ² collapse to + numerical zero — no teeth for the unit-gap tripwire. This builder instead + lays down realistic ξ± power laws and a covariance ~ ``(0.1·|ξ|)²`` (the + probe's recipe), so χ² is O(1)-scale and the raw-vs-shim gap is visible. + + ``xip`` overrides the ξ+ values (perturbation teeth). + """ + rng = np.random.default_rng(seed) + n = N_ANG + theta = np.geomspace(1.0, 250.0, n) # arcmin + z = np.linspace(0.01, 3.0, 200) + nz = z**2 * np.exp(-((z / 0.5) ** 1.5)) + + xip_vals = 2e-4 * (theta / 10.0) ** -0.8 * (1 + 0.05 * rng.standard_normal(n)) + xim_vals = 1e-4 * (theta / 10.0) ** -0.5 * (1 + 0.05 * rng.standard_normal(n)) + if xip is not None: + xip_vals = xip + + s = sacc_io.new_sacc({0: (z, nz)}, {"catalogue_version": "test"}) + sacc_io.add_xi(s, (0, 0), theta, xip_vals, xim_vals, grid="coarse") + + sig = 0.1 * np.abs(np.concatenate([xip_vals, xim_vals])) + a = rng.standard_normal((2 * n, 3 * n)) + cov = (a @ a.T / (3 * n)) * np.outer(sig, sig) * 0.3 + np.diag(sig**2) + s.add_covariance(cov) + return s, theta + + +def _write_pair(tmp_path, seed=0, name="probe", *, xip=None): + """Write a realistic analysis SACC and its PR-3 converter 2pt-FITS. + + Returns ``(sacc_path, fits_path)``. The SACC carries the arcmin ξ± tags the + shim converts; the FITS is the byte-compatible product ``2pt_like`` reads. + """ + s, _theta = _realistic_sacc(seed, xip=xip) + sacc_path = str(tmp_path / f"{name}.sacc") + sacc_io.save(s, sacc_path) + fits_path = str(tmp_path / f"{name}_2pt.fits") + twopoint_convert.sacc_to_twopoint_fits(sacc_io.load(sacc_path), fits_path, n_bins=1) + return sacc_path, fits_path + + +# --------------------------------------------------------------------------- +# 1. Core equality: shim on SACC ≡ 2pt_like on converter FITS +# --------------------------------------------------------------------------- +def test_shimmed_equals_2pt_like_exact(tmp_path, m_shim, m_2pt): + """The shimmed sacc_like and 2pt_like agree exactly on the same data+theory. + + Same synthetic theory block, the SACC through the shim vs the converter FITS + through 2pt_like: χ², log-likelihood and the theory vector must match to + numerical precision (the probe observed exact equality). This is the PR's + central contract — the native path reproduces the validated converter path. + """ + sacc_path, fits_path = _write_pair(tmp_path, seed=0) + + like_s, chi2_s, theory_s, n_s = _run(m_shim, _shim_opts(sacc_path)) + like_t, chi2_t, theory_t, n_t = _run(m_2pt, _twopt_opts(fits_path)) + + assert n_s == n_t == 2 * N_ANG + np.testing.assert_allclose(chi2_s, chi2_t, rtol=1e-12) + np.testing.assert_allclose(like_s, like_t, rtol=1e-12) + np.testing.assert_allclose(theory_s, theory_t, rtol=1e-12) + + +# --------------------------------------------------------------------------- +# 2. Tripwire: raw upstream sacc_like is broken on arcmin tags +# --------------------------------------------------------------------------- +def test_upstream_unit_gap_tripwire(tmp_path, m_raw_sacc, m_2pt): + """RAW upstream sacc_like (no shim) gives a wildly wrong χ² on arcmin tags. + + Documents and guards the arcmin→rad gap the shim fixes: with raw ``theta`` + tags the theory spline is evaluated outside its (radian) grid and returns 0, + collapsing χ² to dᵀC⁻¹d. We assert the relative χ² difference against + 2pt_like exceeds 10 (the probe saw Δχ²≈3184). + + IF THIS TEST EVER FAILS: upstream ``sacc_like`` has fixed its units — the + shim's arcmin→rad conversion is now redundant and the shim can be retired. + """ + sacc_path, fits_path = _write_pair(tmp_path, seed=0) + + _like_r, chi2_raw, _theory_r, _n_r = _run(m_raw_sacc, _shim_opts(sacc_path)) + _like_t, chi2_t, _theory_t, _n_t = _run(m_2pt, _twopt_opts(fits_path)) + + rel = abs(chi2_raw - chi2_t) / abs(chi2_t) + assert rel > 10, ( + f"raw sacc_like χ²={chi2_raw:.6g} is within 10× of 2pt_like χ²={chi2_t:.6g} " + f"(rel diff {rel:.3g}) — the upstream unit gap appears fixed; retire the shim" + ) + + +# --------------------------------------------------------------------------- +# 3. Scale cuts equivalent through both engines +# --------------------------------------------------------------------------- +def test_scale_cuts_equivalent(tmp_path, m_shim, m_2pt): + """The same arcmin scale cuts give the same post-cut N and χ² on both engines. + + Cuts are expressed in the each engine's grammar but the SAME numeric arcmin + values (shim cuts run before the arcmin→rad conversion, so they take arcmin + just like 2pt_like's angle_range). ξ+ ∈ [10, 200], ξ− ∈ [20, 200] arcmin. + """ + sacc_path, fits_path = _write_pair(tmp_path, seed=1) + + shim_cuts = _shim_opts( + sacc_path, + **{ + "angle_range_galaxy_shear_xi_plus_source_0_source_0": np.array( + [10.0, 200.0] + ), + "angle_range_galaxy_shear_xi_minus_source_0_source_0": np.array( + [20.0, 200.0] + ), + }, + ) + twopt_cuts = _twopt_opts( + fits_path, + **{ + "angle_range_XI_PLUS_1_1": np.array([10.0, 200.0]), + "angle_range_XI_MINUS_1_1": np.array([20.0, 200.0]), + }, + ) + + _like_s, chi2_s, _theory_s, n_s = _run(m_shim, shim_cuts) + _like_t, chi2_t, _theory_t, n_t = _run(m_2pt, twopt_cuts) + + assert n_s == n_t + assert n_s < 2 * N_ANG # the cuts actually removed points + np.testing.assert_allclose(chi2_s, chi2_t, rtol=1e-12) + + +# --------------------------------------------------------------------------- +# 4. Perturbation moves both engines identically +# --------------------------------------------------------------------------- +def test_perturbation_moves_both_identically(tmp_path, m_shim, m_2pt): + """Perturbing one data value shifts both engines' χ² by the identical amount. + + Teeth: build the base pair and a pair whose first ξ+ value is bumped, and + require Δχ²(shim) == Δχ²(2pt_like). If either engine ignored the perturbed + point (e.g. a misaligned data vector), the deltas would diverge. + """ + # Base pair, then a pair whose first ξ+ value is bumped. Rebuild the base ξ+ + # from the same seed so only the one perturbed entry differs. + base_s, theta = _realistic_sacc(seed=2) + base_xip = np.array( + [p.value for p in base_s.data if p.data_type == sacc_io.XI_PLUS] + ) + sacc_b, fits_b = _write_pair(tmp_path, seed=2, name="base") + + pert_xip = base_xip.copy() + pert_xip[0] += 5e-5 + sacc_p, fits_p = _write_pair(tmp_path, seed=2, name="pert", xip=pert_xip) + + _l, chi2_shim_b, _t, _n = _run(m_shim, _shim_opts(sacc_b)) + _l, chi2_shim_p, _t, _n = _run(m_shim, _shim_opts(sacc_p)) + _l, chi2_2pt_b, _t, _n = _run(m_2pt, _twopt_opts(fits_b)) + _l, chi2_2pt_p, _t, _n = _run(m_2pt, _twopt_opts(fits_p)) + + d_shim = chi2_shim_p - chi2_shim_b + d_2pt = chi2_2pt_p - chi2_2pt_b + assert abs(d_shim) > 0 # the perturbation actually moved χ² + np.testing.assert_allclose(d_shim, d_2pt, rtol=1e-10) + + +# --------------------------------------------------------------------------- +# 5. Ordering guard raises on a pair-major tomographic file +# --------------------------------------------------------------------------- +def test_ordering_guard_raises_pair_major_tomographic(tmp_path, m_shim): + """A pair-major 2-bin SACC trips the shim's ordering guard at setup. + + Inserting ξ± per pair — (0,0), then (0,1), then (1,1) — lays the data vector + out pair-major ([ξ+;ξ−] per pair), while ``get_data_types()`` groups the + theory loop type-major (all ξ+ pairs, then all ξ− pairs). The two orders + disagree (verified empirically), so the guard must raise a ValueError + mentioning the ordering hazard rather than silently mis-comparing. + """ + theta = np.geomspace(1.0, 250.0, N_ANG) # arcmin + z = np.linspace(0.01, 3.0, 200) + nz = z**2 * np.exp(-((z / 0.5) ** 1.5)) + xip = np.ones(N_ANG) * 1e-4 + xim = np.ones(N_ANG) * 1e-4 + s = sacc_io.new_sacc({0: (z, nz), 1: (z, nz)}) + for pair in [(0, 0), (0, 1), (1, 1)]: + sacc_io.add_xi(s, pair, theta, xip, xim, grid="coarse") + s.add_covariance(np.eye(len(s.mean))) + sacc_path = str(tmp_path / "pair_major.sacc") + sacc_io.save(s, sacc_path) + + with pytest.raises(ValueError, match="ordering"): + m_shim.setup(_as_option_block(_shim_opts(sacc_path))) + + +def _as_option_block(options): + """Build a raw options DataBlock (keys under ``option_section``) for setup().""" + opt = DataBlock() + for key, value in options.items(): + opt[option_section, key] = value + return opt + + +# --------------------------------------------------------------------------- +# 6. theta conversion scoped to real-category types (COSEBIs untouched) +# --------------------------------------------------------------------------- +def test_theta_conversion_scoped_to_real_types(tmp_path, m_shim): + """The shim converts ξ ``theta`` tags but leaves cosebi ``n`` tags untouched. + + Build a SACC carrying both ξ± (real) and COSEBIs (a non-real ``cosebis`` + category with integer ``n`` tags). After the shim's ``build_data``, the ξ + ``theta`` tags must be scaled arcmin→rad (so they equal the original arcmin + values times the conversion factor), and the cosebi ``n`` tags must be + numerically unchanged. + """ + # Build ξ± + COSEBIs, then attach the covariance last (sacc forbids adding + # points after add_covariance). + theta = np.geomspace(1.0, 250.0, N_ANG) # arcmin + z = np.linspace(0.01, 3.0, 200) + nz = z**2 * np.exp(-((z / 0.5) ** 1.5)) + n_modes = 5 + En = np.arange(1.0, n_modes + 1) + Bn = np.arange(1.0, n_modes + 1) * 0.1 + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi( + s, (0, 0), theta, np.ones(N_ANG) * 1e-4, np.ones(N_ANG) * 1e-4, grid="coarse" + ) + sacc_io.add_cosebis(s, (0, 0), En, Bn, scale_cut=(1.0, 250.0)) + s.add_covariance(np.eye(len(s.mean))) + + sacc_path = str(tmp_path / "with_cosebis.sacc") + sacc_io.save(s, sacc_path) + + # data_sets keeps the cosebis in (so we can check its tags survive); cosebi's + # section/category resolve from sacc_like's default_sections, so build_data + # needs no extra ini config. Only setup() runs (build_data); the theory loop + # (which would want a cosebi theory block) runs at execute, not here. + config = m_shim.setup( + _as_option_block( + _shim_opts( + sacc_path, + data_sets=( + "galaxy_shear_xi_plus galaxy_shear_xi_minus " + "galaxy_shear_cosebi_ee galaxy_shear_cosebi_bb" + ), + ) + ) + ) + + xi_thetas = [ + p.tags["theta"] for p in config.sacc_data.data if p.data_type == sacc_io.XI_PLUS + ] + cosebi_ns = [ + p.tags["n"] for p in config.sacc_data.data if p.data_type == sacc_io.COSEBI_EE + ] + # ξ theta converted to radians (original arcmin × ARCMIN_TO_RAD). + np.testing.assert_allclose( + np.sort(xi_thetas), np.sort(theta * ARCMIN_TO_RAD), rtol=1e-12 + ) + # cosebi n tags untouched (still the integer modes 1..n_modes). + np.testing.assert_array_equal(np.sort(cosebi_ns), np.arange(1, n_modes + 1)) + + +# --------------------------------------------------------------------------- +# 7. Real-data equality (candide-gated) +# --------------------------------------------------------------------------- +_REALDATA = ( + Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") + / "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" + / "cosmosis_SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits" +) + + +@pytest.mark.skipif( + not _REALDATA.exists(), reason=f"real 2pt-FITS not on disk: {_REALDATA}" +) +def test_realdata_shim_equals_2pt_like(tmp_path, m_shim, m_2pt): + """On a real product, the shim on its SACC equals 2pt_like on the FITS. + + Builds a ξ-only analysis SACC from the real 2pt-FITS's own ξ± values and + covariance sub-block (via the converter test's ``_sacc_from_2pt_fits``, then + strip to ξ±), writes it, converts it to a plain-ξ FITS, and runs both engines + against the synthetic theory block. Scoping to ξ± isolates the shear + likelihood equality on the true (20-point-per-sign) data-vector shape — the + IA-only inference scope this PR targets — and keeps ``2pt_like``'s + ``twopoint.from_fits`` from tripping over the real file's separate + COVMAT_CELL / τ blocks. + """ + from astropy.io import fits + + from sp_validation.tests.test_twopoint_convert_realdata import _sacc_from_2pt_fits + + with fits.open(_REALDATA) as hdul: + full_s, _rho_hdu, _tau_hdu = _sacc_from_2pt_fits(hdul) + + # Rebuild a ξ-only SACC: same n(z), ξ± values and ξ± covariance sub-block. + source = sacc_io.source_name(0) + z, nz = sacc_io.get_nz(full_s, 0) + theta, xip, xim = sacc_io.get_xi(full_s, (0, 0), grid="coarse") + xi_idx = np.concatenate( + [ + full_s.indices(sacc_io.XI_PLUS, (source, source)), + full_s.indices(sacc_io.XI_MINUS, (source, source)), + ] + ) + xi_cov = full_s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi(s, (0, 0), theta, xip, xim, grid="coarse") + s.add_covariance(xi_cov) + + sacc_path = str(tmp_path / "real_xi.sacc") + sacc_io.save(s, sacc_path) + fits_path = str(tmp_path / "real_xi_2pt.fits") + twopoint_convert.sacc_to_twopoint_fits(sacc_io.load(sacc_path), fits_path, n_bins=1) + + like_s, chi2_s, theory_s, n_s = _run(m_shim, _shim_opts(sacc_path)) + like_t, chi2_t, theory_t, n_t = _run(m_2pt, _twopt_opts(fits_path)) + + assert n_s == n_t + assert n_s == 2 * len(theta) + np.testing.assert_allclose(chi2_s, chi2_t, rtol=1e-10) + np.testing.assert_allclose(like_s, like_t, rtol=1e-10) + np.testing.assert_allclose(theory_s, theory_t, rtol=1e-10) From 1b05bf7016c111df253b782f0ee6dfdf7ce3f58e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 16:35:37 +0200 Subject: [PATCH 28/38] feat(inference): sacc_like ini template + config generator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the CosmoSIS pipeline ini template for the native SACC path (cosmosis_pipeline_A_ia_sacc.ini): copies the A_ia template, swapping load_nz_fits→load_nz_sacc (native n(z) from the SACC's source_i NZ tracers) and 2pt_like→sacc_like (the shim, via SP_VALIDATION_MODULES, csl_dir=COSMOSIS_DIR, like_name=2pt_like for block-key parity). Same numeric scale cuts as A_ia, in the sacc_like angle_range grammar (full data-type names + source_0 pairs). generate_inference_config.py fills a template's [DEFAULT] section with concrete paths (SCRATCH, FITS_FILE|SACC_FILE, COSMOSIS_DIR, SP_VALIDATION_MODULES resolved from sp_validation.__file__). Dual-mode (snakemake object OR argparse CLI) like assemble_sacc.py; plain text processing (the pipeline.sh sed idiom) that REPLACES an existing DEFAULT key in place and prepends only new ones — no configparser round-trip (which would strip comments + %(...)s interpolation) and no duplicate DEFAULT key (which CosmoSIS's strict parser rejects). Tests (no cosmosis needed): both templates fill + interpolate cleanly, the referenced module file exists, no %(...)s placeholder survives, missing-DEFAULT raises. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- .../cosmosis_pipeline_A_ia_sacc.ini | 110 ++++++++++++ .../tests/test_generate_inference_config.py | 138 +++++++++++++++ workflow/scripts/generate_inference_config.py | 159 ++++++++++++++++++ 3 files changed, 407 insertions(+) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini create mode 100644 src/sp_validation/tests/test_generate_inference_config.py create mode 100644 workflow/scripts/generate_inference_config.py diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini new file mode 100644 index 00000000..5c31a476 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia_sacc.ini @@ -0,0 +1,110 @@ +#parameters used elsewhere in this file +[DEFAULT] +COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa + + +[pipeline] +modules = consistency sample_S8 camb load_nz_sacc photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias sacc_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[polychord] +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[test] + +[output] +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_sacc] +file = %(COSMOSIS_DIR)s/number_density/load_nz_sacc/load_nz_sacc.py +nz_file = %(SACC_FILE)s +data_sets = source + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +; Native SACC likelihood via the sp_validation shim (arcmin->rad + ordering +; guard over CosmoSIS's SaccClLikelihood). data_sets/angle ranges use the SACC +; grammar (full data-type names + tracer pairs). like_name=2pt_like keeps the +; block keys identical to the 2pt_like path so chain post-processing is unchanged. +[sacc_like] +file = %(SP_VALIDATION_MODULES)s/sacc_like_unions.py +csl_dir = %(COSMOSIS_DIR)s +data_file = %(SACC_FILE)s +data_sets = galaxy_shear_xi_plus galaxy_shear_xi_minus +like_name = 2pt_like + +angle_range_galaxy_shear_xi_plus_source_0_source_0 = 10.0 200.0 +angle_range_galaxy_shear_xi_minus_source_0_source_0 = 20.0 200.0 diff --git a/src/sp_validation/tests/test_generate_inference_config.py b/src/sp_validation/tests/test_generate_inference_config.py new file mode 100644 index 00000000..4fbbb98d --- /dev/null +++ b/src/sp_validation/tests/test_generate_inference_config.py @@ -0,0 +1,138 @@ +"""Tests for the CosmoSIS inference-config generator. + +The generator (:mod:`workflow.scripts.generate_inference_config`) fills a +pipeline ini template's ``[DEFAULT]`` section with concrete paths so CosmoSIS's +ConfigParser resolves the template's ``%(KEY)s`` placeholders. These tests need +no cosmosis: they check that the substituted DEFAULT keys land, that the module +file paths the templates reference exist on disk, and that no ``%(...)s`` +placeholder is left unresolved after filling. +""" + +import configparser +import importlib.util +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[3] +_SCRIPT = _REPO / "workflow" / "scripts" / "generate_inference_config.py" +_CONFIG_DIR = _REPO / "cosmo_inference" / "cosmosis_config" +_SACC_TEMPLATE = _CONFIG_DIR / "cosmosis_pipeline_A_ia_sacc.ini" +_FITS_TEMPLATE = _CONFIG_DIR / "cosmosis_pipeline_A_ia.ini" + + +def _load_generator(): + spec = importlib.util.spec_from_file_location("gen_inference_cfg", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +gen = _load_generator() + + +def _read_interpolated(ini_path): + """Parse the generated ini with interpolation ON (the way CosmoSIS reads it). + + CosmoSIS uses ``%(KEY)s`` BasicInterpolation with case-*preserved* keys, so + set ``optionxform = str`` (stdlib configparser lowercases keys by default, + which would break the uppercase ``%(FITS_FILE)s`` / ``%(SACC_FILE)s`` + lookups). The default BasicInterpolation then raises if a referenced key is + missing — exactly the failure we want to catch. + """ + parser = configparser.ConfigParser() + parser.optionxform = str + parser.read(ini_path) + return parser + + +def test_sacc_template_defaults_filled(tmp_path): + """The generated sacc ini carries the substituted DEFAULT keys and resolves.""" + out = tmp_path / "gen_sacc.ini" + gen.generate_inference_config( + _SACC_TEMPLATE, + out, + gen._substitutions( + scratch="/scratch/run", + cosmosis_dir="/csl", + sacc_file="/data/v1.sacc", + ), + ) + text = out.read_text() + assert "SCRATCH = /scratch/run" in text + assert "SACC_FILE = /data/v1.sacc" in text + assert "COSMOSIS_DIR = /csl" in text + assert "SP_VALIDATION_MODULES = " in text + # FITS_FILE is None for the sacc path — must be dropped, not written as "None". + assert "FITS_FILE" not in text + + parser = _read_interpolated(out) + # The sacc_like data_file interpolates SACC_FILE; load_nz_sacc too. + assert parser["sacc_like"]["data_file"] == "/data/v1.sacc" + assert parser["load_nz_sacc"]["nz_file"] == "/data/v1.sacc" + assert parser["sacc_like"]["csl_dir"] == "/csl" + + +def test_fits_template_defaults_filled(tmp_path): + """The generated 2pt_like ini carries the substituted DEFAULT keys and resolves.""" + out = tmp_path / "gen_fits.ini" + gen.generate_inference_config( + _FITS_TEMPLATE, + out, + gen._substitutions( + scratch="/scratch/run", + cosmosis_dir="/csl", + fits_file="/data/v1.fits", + ), + ) + text = out.read_text() + assert "SCRATCH = /scratch/run" in text + assert "FITS_FILE = /data/v1.fits" in text + assert "SACC_FILE" not in text + + parser = _read_interpolated(out) + assert parser["2pt_like"]["data_file"] == "/data/v1.fits" + assert parser["load_nz_fits"]["nz_file"] == "/data/v1.fits" + + +def test_sacc_template_module_file_exists(): + """The sacc_like module file the generated ini points at exists on disk. + + ``SP_VALIDATION_MODULES`` resolves to the installed package dir; + ``sacc_like_unions.py`` must live there (it is the shim CosmoSIS loads). + """ + modules = Path(gen._sp_validation_modules()) + assert (modules / "sacc_like_unions.py").is_file() + + +def test_all_placeholders_resolve(tmp_path): + """No ``%(...)s`` placeholder survives interpolation in either template. + + A missing DEFAULT key would make ConfigParser raise on access; iterate every + option in every section to force resolution of all placeholders. + """ + for template, subs in ( + ( + _SACC_TEMPLATE, + gen._substitutions(scratch="/s", cosmosis_dir="/csl", sacc_file="/d.sacc"), + ), + ( + _FITS_TEMPLATE, + gen._substitutions(scratch="/s", cosmosis_dir="/csl", fits_file="/d.fits"), + ), + ): + out = tmp_path / (template.stem + ".gen.ini") + gen.generate_inference_config(template, out, subs) + parser = _read_interpolated(out) + for section in parser.sections(): + for key in parser[section]: + value = parser[section][key] # raises if a placeholder is unresolved + assert "%(" not in value, f"[{section}] {key} = {value}" + + +def test_no_default_section_raises(tmp_path): + """A template with no [DEFAULT] section is a loud error.""" + bad = tmp_path / "bad.ini" + bad.write_text("[pipeline]\nmodules = a b c\n") + with pytest.raises(ValueError, match="DEFAULT"): + gen.generate_inference_config(bad, tmp_path / "out.ini", {"SCRATCH": "/s"}) diff --git a/workflow/scripts/generate_inference_config.py b/workflow/scripts/generate_inference_config.py new file mode 100644 index 00000000..68a8efe7 --- /dev/null +++ b/workflow/scripts/generate_inference_config.py @@ -0,0 +1,159 @@ +"""Generate a CosmoSIS pipeline ini by filling a template's ``[DEFAULT]`` section. + +Dual-mode, like ``assemble_sacc.py``. Under Snakemake (``script:`` directive) the +injected ``snakemake`` object supplies the template, output path and DEFAULT +substitutions; as a standalone CLI (argparse) the same fill runs from explicit +flags. + +The template carries ``%(KEY)s`` interpolation placeholders (SCRATCH, FITS_FILE +or SACC_FILE, COSMOSIS_DIR, SP_VALIDATION_MODULES) in its module sections; this +script prepends the concrete ``KEY = value`` lines into ``[DEFAULT]`` so +CosmoSIS's ConfigParser resolves them at load. It is deliberately plain text +processing — appending lines after the ``[DEFAULT]`` header, the same idiom as +``pipeline.sh``'s ``sed -i "/^\\[DEFAULT\\]/a\\KEY = value"`` — rather than a +configparser round-trip, which would strip the template's comments and its +``%(...)s`` interpolation. + +``SP_VALIDATION_MODULES`` is resolved from ``sp_validation.__file__``'s parent so +the generated ini points at the installed package's module directory (where +``sacc_like_unions.py`` lives) regardless of checkout location. +""" + +import argparse +from pathlib import Path + + +def _sp_validation_modules(): + """The directory holding the sp_validation CosmoSIS module files. + + Resolved from the installed package so the generated ini finds + ``sacc_like_unions.py`` wherever sp_validation is installed. + """ + import sp_validation + + return str(Path(sp_validation.__file__).resolve().parent) + + +def generate_inference_config(template_path, out_path, substitutions): + """Write ``out_path`` from ``template_path`` with ``substitutions`` in DEFAULT. + + Parameters + ---------- + template_path : str or Path + The pipeline ini template (carries ``%(KEY)s`` placeholders). + out_path : str or Path + Destination ini. + substitutions : dict + ``{KEY: value}`` lines prepended into the template's ``[DEFAULT]`` + section. Every referenced ``%(KEY)s`` in the template must have a value + here (COSMOSIS_DIR already sits in the template's DEFAULT and may be + overridden). ``None`` values are dropped (an absent optional key). + """ + lines = Path(template_path).read_text().splitlines(keepends=True) + + header = "[DEFAULT]" + default_idx = next( + (i for i, line in enumerate(lines) if line.strip() == header), None + ) + if default_idx is None: + raise ValueError(f"template {template_path} has no [DEFAULT] section") + + # The end of the DEFAULT section: the next `[section]` header, or EOF. + section_end = next( + ( + i + for i in range(default_idx + 1, len(lines)) + if lines[i].lstrip().startswith("[") + ), + len(lines), + ) + + # A key the template already declares in DEFAULT is REPLACED in place (e.g. the + # template's placeholder COSMOSIS_DIR); a genuinely-new key is prepended just + # after the header. This avoids a duplicate DEFAULT key, which CosmoSIS's + # ConfigParser (strict) rejects. + wanted = {key: value for key, value in substitutions.items() if value is not None} + remaining = dict(wanted) + for i in range(default_idx + 1, section_end): + stripped = lines[i].lstrip() + if not stripped or stripped.startswith(("#", ";", "[")): + continue + existing_key = stripped.split("=", 1)[0].strip() + if existing_key in remaining: + lines[i] = f"{existing_key} = {remaining.pop(existing_key)}\n" + + prepended = [f"{key} = {value}\n" for key, value in remaining.items()] + out_lines = lines[: default_idx + 1] + prepended + lines[default_idx + 1 :] + + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text("".join(out_lines)) + print( + f"Wrote {out_path} from {template_path} " + f"({len(wanted)} DEFAULT keys, {len(prepended)} new)" + ) + return str(out_path) + + +def _substitutions(scratch, cosmosis_dir, *, fits_file=None, sacc_file=None): + """Assemble the DEFAULT substitution dict, resolving SP_VALIDATION_MODULES. + + ``fits_file`` (2pt_like path) and ``sacc_file`` (sacc_like path) are mutually + the data-file placeholder for their template; whichever the template + references is filled, the other left absent. + """ + return { + "SCRATCH": scratch, + "FITS_FILE": fits_file, + "SACC_FILE": sacc_file, + "COSMOSIS_DIR": cosmosis_dir, + "SP_VALIDATION_MODULES": _sp_validation_modules(), + } + + +def _from_snakemake(smk): + p = smk.params + generate_inference_config( + template_path=smk.input[0] + if not hasattr(smk.input, "template") + else smk.input.template, + out_path=str(smk.output[0]), + substitutions=_substitutions( + scratch=p["scratch"], + cosmosis_dir=p["cosmosis_dir"], + fits_file=p.get("fits_file", None), + sacc_file=p.get("sacc_file", None), + ), + ) + + +def _from_cli(argv=None): + ap = argparse.ArgumentParser( + description="Generate a CosmoSIS pipeline ini from a template + DEFAULT subs." + ) + ap.add_argument("--template", required=True, help="Pipeline ini template") + ap.add_argument("--out", required=True, help="Output ini path") + ap.add_argument("--scratch", required=True, help="SCRATCH value") + ap.add_argument("--cosmosis-dir", required=True, help="COSMOSIS_DIR value") + ap.add_argument("--fits-file", default=None, help="FITS_FILE (2pt_like path)") + ap.add_argument("--sacc-file", default=None, help="SACC_FILE (sacc_like path)") + a = ap.parse_args(argv) + generate_inference_config( + template_path=a.template, + out_path=a.out, + substitutions=_substitutions( + scratch=a.scratch, + cosmosis_dir=a.cosmosis_dir, + fits_file=a.fits_file, + sacc_file=a.sacc_file, + ), + ) + + +if __name__ == "__main__": + try: + snakemake # noqa: F821 — injected by Snakemake's script: directive + except NameError: + _from_cli() + else: + _from_snakemake(snakemake) # noqa: F821 From 5d123508d64ce892009b655896de56389c9fe9bd Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 16:35:53 +0200 Subject: [PATCH 29/38] feat(inference): rewire inference_prep to consume the assembled SACC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revive the real-data inference_prep from its dormant, pre-SACC state. It now consumes the assembled analysis {version}.sacc (cosmo_val.smk's assemble_sacc, bound lazily through cv_analysis_sacc) and emits the A_ia (IA-only, ξ±) file-prep products: (a) the converter 2pt-FITS (sacc_to_twopoint_fits, pure ξ — no rho/tau sidecars, A_ia scope) + a generated 2pt_like ini (the validating/legacy path), and (b) a generated sacc_like ini pointing at the SACC (the native path, validated bit-for-bit against (a)). The cosmosis_fitting.py real-data assembly is retired from this rule; the glass-mock rules keep it (their SACC migration is out of scope, noted inline). inference_fiducial extends to all three prep outputs. CSL_DIR is read lazily (inference.smk is parsed by every paper workflow, but only cosmo_val carries inference.csl_dir) — a missing key still fails loudly, just at DAG time rather than at parse time for an unrelated (bmodes) workflow. config.yaml: add inference.csl_dir. The workflow dry-run guard gains a test covering the revived inference_prep DAG (SACC in, converter FITS + both engine inis out, no cosmosis_fitting.py real-data assembly). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- papers/cosmo_val/config/config.yaml | 5 +- .../tests/test_bmodes_workflow_dry_run.py | 25 +++ workflow/rules/inference.smk | 166 ++++++++++-------- 3 files changed, 122 insertions(+), 74 deletions(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 945f2f8f..a7f19a8d 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -116,11 +116,14 @@ harmonic: binning: powspace nbins: 32 -# Cosmological inference data-product locations (dormant subsystem). +# Cosmological inference data-product locations + tooling. inference: chains_dir: "/n09data/guerrini/output_chains" glass_mock_data_dir: "/n09data/guerrini/glass_mock_v1.4.6/results" glass_mock_chains_dir: "/n09data/guerrini/glass_mock_chains" + # CosmoSIS Standard Library checkout — fills COSMOSIS_DIR in the generated + # pipeline inis (the module `file =` paths and the sacc_like shim's csl_dir). + csl_dir: "/n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa" cosebis: theta_min: 1.0 diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index 87afc416..b9486b5d 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -96,3 +96,28 @@ def test_cosmo_val_workflow_assemble_dry_runs(): assert f"pseudo_cl_cov_{version}_blind=A_powspace_nbins=32.fits" in out, out for part in ("_xi_coarse_", "_cosebis.sacc", "_pure_eb.sacc", "rho_tau_"): assert part in out, f"missing {part} part in assemble DAG:\n{out}" + + +@requires_candide_data +def test_cosmo_val_inference_prep_dry_runs(): + """The revived (PR-7) inference_prep DAG resolves end to end from the SACC. + + inference_fiducial must pull inference_prep, which consumes the assembled + {version}.sacc and emits the converter 2pt-FITS plus BOTH generated pipeline + inis (2pt_like and the native sacc_like). The old cosmosis_fitting.py real- + data assembly is retired from this path; the glass-mock rules keep it. + """ + version = "SP_v1.4.6.3_leak_corr" + result = _dry_run(_repo_root() / "papers/cosmo_val", ["inference_fiducial"]) + assert result.returncode == 0, result.stdout + out = result.stdout + assert "rule inference_prep:" in out, out + assert "rule inference_fiducial:" in out, out + # inference_prep consumes the assembled analysis SACC (not per-sign xi FITS). + assert f"{version}.sacc" in out, out + # It emits the converter FITS + both engine inis. + assert f"cosmosis_{version}.fits" in out, out + assert f"cosmosis_pipeline_{version}_A_ia.ini" in out, out + assert f"cosmosis_pipeline_{version}_A_ia_sacc.ini" in out, out + # The retired real-data assembly script must not appear in this DAG's prep. + assert "cosmosis_fitting.py --cosmosis-root" not in out, out diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index 6bf68645..9c73c52a 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -1,15 +1,19 @@ -# Imports from Snakefile: FIDUCIAL, COSMO_INFERENCE, COSMO_VAL, covariance_path, build_redshift_path, fiducial_binning_suffix -# NOTE: dormant subsystem. The file-name plumbing (config-driven paths + the -# producer-tagged pseudo-Cl names) is fixed and the DAG is valid, but it has not -# been run end-to-end. Reviving it still needs the FITS-CONTENT plumbing -# reconciled: cosmosis_fitting.py reads ELL/EE/BB + COVAR_FULL, while the -# producers write PSEUDO_CELL/ELL + COVAR_BB_BB. +# Imports from common (via `from common import *`): FIDUCIAL, COSMO_INFERENCE, +# COSMO_VAL, WORKFLOW_SCRIPTS, covariance_path, build_redshift_path, +# fiducial_binning_suffix. cv_analysis_sacc arrives from cosmo_val.smk (resolved +# lazily at DAG time, since that file is included after this one). +# +# Two paths live here: +# * Real-data inference_prep — LIVE (PR 7): consumes the assembled {version}.sacc +# and emits the converter 2pt-FITS + both engine inis (2pt_like, sacc_like). +# * glass-mock rules — still cosmosis_fitting.py-based (their SACC migration is +# out of scope); the pseudo-Cl file-name plumbing they depend on stays below. # Output root for CosmoSIS data products + configs. COSMO_INFERENCE (common.py) # already resolves to THIS repo's cosmo_inference dir, so the products land # beside the code that builds them rather than in a contributor's home. COSMO_INFERENCE_PROD = COSMO_INFERENCE -# Working directory for the cosmosis_fitting.py invocation — the same repo dir. +# Working directory for the (glass-mock) cosmosis_fitting.py invocation. COSMO_INFERENCE_RUNDIR = str(COSMO_INFERENCE) # External chain/mock locations are deployment-specific, so they live in config. @@ -56,88 +60,104 @@ def pseudo_cl_assets(version): return str(cl_path), str(cov_path) # --------------------------------------------------------------------------- -# DORMANT — pre-SACC cosmosis assembly. Migration to native SACC deferred to -# PR 7 (native-SACC inference consumption); do NOT deep-migrate here. +# Real-data inference prep — LIVE (native SACC, PR 7). Consumes the assembled +# analysis {version}.sacc (cosmo_val.smk's assemble_sacc rule) and emits the two +# file-prep products the A_ia (IA-only, ξ±) fiducial pipeline needs: +# (a) the converter 2pt-FITS (sacc_to_twopoint_fits) + a generated 2pt_like ini +# — the validating/legacy path (retiring cosmosis_fitting.py's assembly), +# (b) a generated sacc_like ini pointing at the SACC directly — the native path +# validated bit-for-bit against (a) (test_sacc_like.py). +# The converter is A_ia-scoped: no rho/tau sidecars, so it emits a pure-ξ FITS +# (it ignores the SACC's extra data types). This is file-prep only — the actual +# CosmoSIS sampling still runs via pipeline.sh against these products. # -# The SACC migration (PR 4) removed the data products several of these inputs -# name, so this rule's DAG no longer resolves and is NOT reachable from the -# cosmo_val suite (cosmo_val_all never requests it). Stale inputs: -# - xi_plus / xi_minus FITS: the `xi` rule now emits the coarse ξ± SACC part -# ({version}_xi_coarse_...sacc), not per-sign FITS. -# - pseudo_cl / pseudo_cl_cov via pseudo_cl_assets(): the `pseudo_cl` rule now -# writes .sacc (pseudo_cl_assets still requests .fits). -# PR 7 rewires this to consume the assembled {version}.sacc (built by -# cosmo_val.smk's assemble_sacc rule) directly, retiring cosmosis_fitting.py's -# per-product FITS assembly. Until then the inference target is knowingly red. +# The glass-mock rules below stay cosmosis_fitting.py-based; their SACC migration +# is out of scope for PR 7. # --------------------------------------------------------------------------- +INFERENCE_TEMPLATES = COSMO_INFERENCE_PROD / "cosmosis_config" + + +def _csl_dir(): + """The CSL checkout that fills COSMOSIS_DIR / sacc_like csl_dir in the inis. + + Read lazily (at DAG time, inside inference_prep's params) rather than at + module parse time: inference.smk is included by every paper workflow, but + only papers that run inference (cosmo_val) carry inference.csl_dir. A missing + key still fails loudly — just when the real-data inference is actually built, + not when an unrelated (bmodes) workflow merely parses this file. + """ + return INFERENCE["csl_dir"] + + rule inference_prep: input: - # Processed covariance matrix - use centralized covariance_path() - cov_matrix=lambda w: covariance_path(w.version, w.blind, min_sep=w.min_sep, max_sep=w.max_sep, nbins=w.nbins), - # Xi FITS files — PRE-SACC (no longer produced; see dormant note above) - xi_plus=str(COSMO_VAL / "xi_plus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - xi_minus=str(COSMO_VAL / "xi_minus_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # n(z) file (using new location with base version mapping) - nz_file=lambda w: build_redshift_path(w.version, w.blind), - # rho/tau stats - rho_stats=str(COSMO_VAL / "rho_tau_stats/rho_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - tau_stats=str(COSMO_VAL / "rho_tau_stats/tau_stats_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits"), - # tau covariance (tracked as dependency) - tau_cov=str(COSMO_VAL / "rho_tau_stats/cov_tau_{version}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}_th.npy"), - # pseudo_cl / pseudo_cl_cov — PRE-SACC (.fits path; producer now writes .sacc) - pseudo_cl=lambda w: pseudo_cl_assets(w.version)[0], - pseudo_cl_cov=lambda w: pseudo_cl_assets(w.version)[1], + # The terminal assembled analysis SACC (cosmo_val.smk assemble_sacc). Bound + # lazily through its helper so the filename tracks that rule, not a literal. + sacc=lambda w: cv_analysis_sacc(w.version), output: - fits_file=str( - COSMO_INFERENCE_PROD - / "data/{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}/cosmosis_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.fits" + fits_file=str(COSMO_INFERENCE_PROD / "data/{version}/cosmosis_{version}.fits"), + config_file_2pt=str( + INFERENCE_TEMPLATES / "cosmosis_pipeline_{version}_A_ia.ini" + ), + config_file_sacc=str( + INFERENCE_TEMPLATES / "cosmosis_pipeline_{version}_A_ia_sacc.ini" ), - config_file=str( - COSMO_INFERENCE_PROD - / "cosmosis_config/cosmosis_pipeline_{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}.ini" - ) params: - cosmosis_root="{version}_{blind}_minsep={min_sep}_maxsep={max_sep}_nbins={nbins}_npatch={npatch}", - data_dir=f"{CHAINS_DIR}/{{version}}_{{blind}}_minsep={{min_sep}}_maxsep={{max_sep}}_nbins={{nbins}}_npatch={{npatch}}", - output_root=str(COSMO_INFERENCE_PROD), + # SCRATCH = the per-version chain output root the generated inis point at. + scratch=lambda w: f"{CHAINS_DIR}/{w.version}", + cosmosis_dir=lambda w: _csl_dir(), + template_2pt=str(INFERENCE_TEMPLATES / "cosmosis_pipeline_A_ia.ini"), + template_sacc=str(INFERENCE_TEMPLATES / "cosmosis_pipeline_A_ia_sacc.ini"), threads: 1 resources: mem_mb=8000, runtime=10, - shell: - """ - cd {COSMO_INFERENCE_RUNDIR} + run: + import os + import sys - # Run inference preparation step with cosmosis_fitting.py - python scripts/cosmosis_fitting.py \ - --cosmosis-root {params.cosmosis_root} \ - --nz-file {input.nz_file} \ - --data-dir {params.data_dir} \ - --output-root {params.output_root} \ - --xi {input.xi_plus} {input.xi_minus} \ - --cov-xi {input.cov_matrix} \ - --use-rho-tau \ - --rho-stats {input.rho_stats} \ - --tau-stats {input.tau_stats} \ - --cov-tau {input.tau_cov} \ - --cl-file {input.pseudo_cl} \ - --cov-cl {input.pseudo_cl_cov} - """ + from sp_validation import sacc_io + from sp_validation.twopoint_convert import sacc_to_twopoint_fits + + os.makedirs(os.path.dirname(output.fits_file), exist_ok=True) + + # (a) converter 2pt-FITS — pure ξ (A_ia scope; no rho/tau sidecars). + sacc_to_twopoint_fits(sacc_io.load(input.sacc), output.fits_file, n_bins=1) + + # (b) + (c) the two generated pipeline inis, from the existing templates. + # WORKFLOW_SCRIPTS (common.py) is the absolute generic-workflow scripts dir. + sys.path.insert(0, WORKFLOW_SCRIPTS) + from generate_inference_config import ( + _substitutions, + generate_inference_config, + ) + + generate_inference_config( + params.template_2pt, + output.config_file_2pt, + _substitutions( + scratch=params.scratch, + cosmosis_dir=params.cosmosis_dir, + fits_file=output.fits_file, + ), + ) + generate_inference_config( + params.template_sacc, + output.config_file_sacc, + _substitutions( + scratch=params.scratch, + cosmosis_dir=params.cosmosis_dir, + sacc_file=input.sacc, + ), + ) rule inference_fiducial: input: - # Use the same output patterns as inference_prep with FIDUCIAL params - rules.inference_prep.output.fits_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], - min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], - nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] - ), - rules.inference_prep.output.config_file.format( - version=FIDUCIAL["version"], blind=FIDUCIAL["blind"], - min_sep=FIDUCIAL["min_sep"], max_sep=FIDUCIAL["max_sep"], - nbins=FIDUCIAL["nbins"], npatch=FIDUCIAL["npatch"] - ) + # The fiducial version's prep products (both engine inis + the FITS). + rules.inference_prep.output.fits_file.format(version=FIDUCIAL["version"]), + rules.inference_prep.output.config_file_2pt.format(version=FIDUCIAL["version"]), + rules.inference_prep.output.config_file_sacc.format(version=FIDUCIAL["version"]), rule inference_glass_mocks: From db286022edbc5810f6f47059fa07136f5a18ca9b Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 17:07:17 +0200 Subject: [PATCH 30/38] fix(sacc_like): keep self.sacc_data in arcmin so save_theory writes arcmin tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding (MEDIUM): the shim converted theta tags arcmin→rad in place on self.sacc_data. Upstream do_likelihood's save_theory / save_realization paths copy self.sacc_data and overwrite only point values, so they were writing radian theta tags into the saved SACC — any arcmin-assuming consumer (sacc_io.get_xi, or re-ingesting the file as data_file, which double-converts to ~8.5e-8) is then silently off by 3437×. Keep self.sacc_data in arcmin always. build_data now builds a separate self._sacc_data_rad copy with the real-category theta tags converted, and extract_theory_points swaps it in around super().extract_theory_points(block) in a try/finally — so the theory spline sees radians while everything after (save_theory / save_realization) sees the untouched arcmin original. The ordering guard runs on the arcmin object (ordering is unit-independent). Theory equality is unchanged (the swap produces identical results): synthetic χ²=1.18829133107, real-data χ²=420310.753739, both Δχ²=0 vs 2pt_like. New test: run with save_theory set, reload, assert the saved theta tags equal the input file's arcmin tags exactly (and are NOT the radian conversion) and the saved values equal the theory vector; assert a second execute() yields the identical χ² (guards against a double-conversion regression). The theta-scoping test now checks the rad copy carries radians AND the original stays arcmin. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- src/sp_validation/sacc_like_unions.py | 50 ++++++++++++--- src/sp_validation/tests/test_sacc_like.py | 76 ++++++++++++++++++++--- 2 files changed, 108 insertions(+), 18 deletions(-) diff --git a/src/sp_validation/sacc_like_unions.py b/src/sp_validation/sacc_like_unions.py index 2b7e5cc6..5aa610ec 100644 --- a/src/sp_validation/sacc_like_unions.py +++ b/src/sp_validation/sacc_like_unions.py @@ -23,7 +23,10 @@ ``sacc_like`` never does, so the spline is evaluated ~3437× outside its grid, ``SpectrumInterp`` returns 0 there, and χ² silently collapses to dᵀC⁻¹d. The only prior use was the ℓ-space (unit-free) Cℓ path, which is why it was never - caught. We convert the ``theta`` tags to radians here, after scale cuts. + caught. We evaluate the theory against a radian-θ *copy* of the loaded SACC, + keeping ``self.sacc_data`` in arcmin so upstream's save_theory / + save_realization paths (which copy it and overwrite only values) never write + radian tags into a file downstream consumers read as arcmin. 2. **Theory↔data ordering is assumed, never enforced.** The data vector is ``sacc.get_mean()`` (insertion order); the theory vector is @@ -100,29 +103,56 @@ def build_data(self): # (unit-independent) data vector we pass straight through. x, data_vector = super().build_data() - self._convert_real_theta_tags_to_radians() + # self.sacc_data STAYS in arcmin — save_theory / save_realization copy + # it and only overwrite point values, so its θ tags must remain the + # units the file was written in (any consumer, incl. re-ingesting the + # saved SACC as a data_file, assumes arcmin). The radian conversion the + # theory spline needs lives on a separate copy, swapped in only for the + # extraction (see extract_theory_points). + self._sacc_data_rad = self._radian_theta_copy(self.sacc_data) self._assert_theory_order_matches_data() return x, data_vector - def _convert_real_theta_tags_to_radians(self): - """Scale the ``theta`` tag arcmin→rad for every ``real``-category point. + def _radian_theta_copy(self, sacc_data): + """A copy of ``sacc_data`` with ``real``-category θ tags in radians. The theory spline is built on ``block[section, "theta"]`` in radians, - so the ``theta`` tag each point is evaluated at must be radians too. - Scoped to data types whose category (``sections_for_names[dt][0]``) is - ``real``: cosebis ``n`` tags and spectrum ``ell`` tags are unit-free - and must not be touched, and only the ``theta`` tag is converted - (``theta_nom`` etc. are metadata the likelihood never evaluates). + so the ``theta`` tag each point is evaluated against must be radians + too. Scoped to data types whose category + (``sections_for_names[dt][0]``) is ``real``: cosebis ``n`` tags and + spectrum ``ell`` tags are unit-free and left untouched, and only the + ``theta`` tag is scaled (``theta_nom`` etc. are metadata the likelihood + never evaluates). Operates on a ``.copy()`` so the original stays + arcmin for the save paths. """ real_types = { dt for dt, (category, _section) in self.sections_for_names.items() if category == "real" } - for point in self.sacc_data.data: + converted = sacc_data.copy() + for point in converted.data: if point.data_type in real_types and "theta" in point.tags: point.tags["theta"] = point.tags["theta"] * ARCMIN_TO_RAD + return converted + + def extract_theory_points(self, block): + """Extract theory against the radian-θ copy, then restore the original. + + Upstream ``extract_theory_points`` reads ``self.sacc_data`` (the θ tag + per point) to evaluate the theory spline; that read needs radians. + Swap in ``self._sacc_data_rad`` for the duration of the upstream call + and restore in ``finally`` so everything else — including the + save_theory / save_realization copies that run afterward in + ``do_likelihood`` — sees the untouched arcmin ``self.sacc_data``. + """ + original = self.sacc_data + self.sacc_data = self._sacc_data_rad + try: + return super().extract_theory_points(block) + finally: + self.sacc_data = original def _assert_theory_order_matches_data(self): """Require the theory-loop order to equal the data-vector order. diff --git a/src/sp_validation/tests/test_sacc_like.py b/src/sp_validation/tests/test_sacc_like.py index f8c6c3b2..8e8c1332 100644 --- a/src/sp_validation/tests/test_sacc_like.py +++ b/src/sp_validation/tests/test_sacc_like.py @@ -429,18 +429,78 @@ def test_theta_conversion_scoped_to_real_types(tmp_path, m_shim): ) ) - xi_thetas = [ - p.tags["theta"] for p in config.sacc_data.data if p.data_type == sacc_io.XI_PLUS + # The conversion lives on the radian copy (_sacc_data_rad); the original + # self.sacc_data stays arcmin (so save_theory writes arcmin tags — Finding 1). + rad_xi_thetas = [ + p.tags["theta"] + for p in config._sacc_data_rad.data + if p.data_type == sacc_io.XI_PLUS + ] + rad_cosebi_ns = [ + p.tags["n"] + for p in config._sacc_data_rad.data + if p.data_type == sacc_io.COSEBI_EE ] - cosebi_ns = [ - p.tags["n"] for p in config.sacc_data.data if p.data_type == sacc_io.COSEBI_EE + orig_xi_thetas = [ + p.tags["theta"] for p in config.sacc_data.data if p.data_type == sacc_io.XI_PLUS ] - # ξ theta converted to radians (original arcmin × ARCMIN_TO_RAD). + # rad copy: ξ theta scaled to radians (original arcmin × ARCMIN_TO_RAD). np.testing.assert_allclose( - np.sort(xi_thetas), np.sort(theta * ARCMIN_TO_RAD), rtol=1e-12 + np.sort(rad_xi_thetas), np.sort(theta * ARCMIN_TO_RAD), rtol=1e-12 ) - # cosebi n tags untouched (still the integer modes 1..n_modes). - np.testing.assert_array_equal(np.sort(cosebi_ns), np.arange(1, n_modes + 1)) + # rad copy: cosebi n tags untouched (still integer modes 1..n_modes). + np.testing.assert_array_equal(np.sort(rad_cosebi_ns), np.arange(1, n_modes + 1)) + # original sacc_data: ξ theta still in arcmin (unmutated). + np.testing.assert_allclose(np.sort(orig_xi_thetas), np.sort(theta), rtol=1e-12) + + +# --------------------------------------------------------------------------- +# 6b. save_theory writes arcmin tags, and re-execute is stable (Finding 1) +# --------------------------------------------------------------------------- +def test_save_theory_writes_arcmin_and_reexecute_stable(tmp_path, m_shim): + """save_theory must write a SACC whose θ tags are still arcmin, not radians. + + Finding 1: because upstream save_theory copies self.sacc_data and overwrites + only point values, self.sacc_data must stay arcmin — otherwise the saved file + carries radian θ tags and any arcmin-assuming consumer (sacc_io.get_xi, or + re-ingesting it as a data_file, which would double-convert to ~8.5e-8) is + silently off by 3437×. Assert the saved θ tags match the input file's tags + exactly (arcmin) and that the saved values equal the theory vector. Also run + execute() twice and require identical χ² — a guard against any accidental + double-conversion creeping back in. + """ + s, theta = _realistic_sacc(seed=5) + sacc_path = str(tmp_path / "in.sacc") + sacc_io.save(s, sacc_path) + input_theta = np.array( + [p.tags["theta"] for p in sacc_io.load(sacc_path).data if "theta" in p.tags] + ) + + save_path = str(tmp_path / "saved_theory.sacc") + opt = _as_option_block(_shim_opts(sacc_path, save_theory=save_path)) + config = m_shim.setup(opt) + + block1 = _theory_block() + m_shim.execute(block1, config) + chi2_1 = block1["data_vector", f"{LIKE_NAME}_CHI2"] + theory = np.asarray(block1["data_vector", f"{LIKE_NAME}_theory"]) + + saved = sacc_io.load(save_path) + saved_theta = np.array([p.tags["theta"] for p in saved.data if "theta" in p.tags]) + saved_values = np.array(saved.mean) + + # θ tags in the saved file are arcmin — identical to the input file's tags. + np.testing.assert_array_equal(saved_theta, input_theta) + # and are NOT the radian conversion (guards against the leak explicitly). + assert not np.allclose(saved_theta, input_theta * ARCMIN_TO_RAD) + # saved values are the theory vector (save_theory overwrites values in order). + np.testing.assert_allclose(saved_values, theory, rtol=1e-12) + + # A second execute() yields the identical χ² — no cumulative mutation. + block2 = _theory_block() + m_shim.execute(block2, config) + chi2_2 = block2["data_vector", f"{LIKE_NAME}_CHI2"] + np.testing.assert_allclose(chi2_2, chi2_1, rtol=1e-12) # --------------------------------------------------------------------------- From becfeb2f328d15ecfb27f54b47be7ef9b0ce1ee1 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 17:07:30 +0200 Subject: [PATCH 31/38] fix(inference): bind ini templates as inference_prep inputs, not params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review finding (MEDIUM): the two pipeline ini templates were referenced as params, so there was no DAG edge — editing a template never regenerated the per-version config files. Bind both as rule inputs (referenced via input.* in the run block). The templates are source files, so anchor them on the running checkout (INFERENCE_TEMPLATE_DIR, off the workflow dir's parent) rather than the env-overridable COSMO_INFERENCE output root; the generated per-version configs still land in COSMO_INFERENCE (INFERENCE_CONFIG_OUT). In a normal run the two roots coincide; the split is what lets a template edit in this checkout drive the DAG. The dry-run guard now asserts both templates appear as inference_prep inputs. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- .../tests/test_bmodes_workflow_dry_run.py | 4 +++ workflow/rules/inference.smk | 27 +++++++++++++------ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py index b9486b5d..91a1923d 100644 --- a/src/sp_validation/tests/test_bmodes_workflow_dry_run.py +++ b/src/sp_validation/tests/test_bmodes_workflow_dry_run.py @@ -115,6 +115,10 @@ def test_cosmo_val_inference_prep_dry_runs(): assert "rule inference_fiducial:" in out, out # inference_prep consumes the assembled analysis SACC (not per-sign xi FITS). assert f"{version}.sacc" in out, out + # ...and both ini TEMPLATES, bound as inputs so a template edit regenerates the + # configs (Finding 2 — a template as params gave no DAG edge). + assert "cosmosis_pipeline_A_ia.ini" in out, out + assert "cosmosis_pipeline_A_ia_sacc.ini" in out, out # It emits the converter FITS + both engine inis. assert f"cosmosis_{version}.fits" in out, out assert f"cosmosis_pipeline_{version}_A_ia.ini" in out, out diff --git a/workflow/rules/inference.smk b/workflow/rules/inference.smk index 9c73c52a..ab7b456c 100644 --- a/workflow/rules/inference.smk +++ b/workflow/rules/inference.smk @@ -74,7 +74,15 @@ def pseudo_cl_assets(version): # The glass-mock rules below stay cosmosis_fitting.py-based; their SACC migration # is out of scope for PR 7. # --------------------------------------------------------------------------- -INFERENCE_TEMPLATES = COSMO_INFERENCE_PROD / "cosmosis_config" +# Generated per-version configs land in the (env-overridable) output root. +INFERENCE_CONFIG_OUT = COSMO_INFERENCE_PROD / "cosmosis_config" +# The ini TEMPLATES are source files: anchor them on the running checkout (repo +# root = the workflow dir's parent, via WORKFLOW_SCRIPTS), NOT on the output root +# — so a template edit in this checkout drives the DAG even when COSMO_INFERENCE +# points elsewhere. In a normal (non-worktree) run the two roots coincide. +INFERENCE_TEMPLATE_DIR = ( + Path(os.path.dirname(WORKFLOW_SCRIPTS)).parent / "cosmo_inference" / "cosmosis_config" +) def _csl_dir(): @@ -94,20 +102,23 @@ rule inference_prep: # The terminal assembled analysis SACC (cosmo_val.smk assemble_sacc). Bound # lazily through its helper so the filename tracks that rule, not a literal. sacc=lambda w: cv_analysis_sacc(w.version), + # The two pipeline ini templates are static repo files, but binding them as + # inputs (not params) puts them in the DAG, so editing a template + # regenerates the configs rather than leaving stale output on disk. + template_2pt=str(INFERENCE_TEMPLATE_DIR / "cosmosis_pipeline_A_ia.ini"), + template_sacc=str(INFERENCE_TEMPLATE_DIR / "cosmosis_pipeline_A_ia_sacc.ini"), output: fits_file=str(COSMO_INFERENCE_PROD / "data/{version}/cosmosis_{version}.fits"), config_file_2pt=str( - INFERENCE_TEMPLATES / "cosmosis_pipeline_{version}_A_ia.ini" + INFERENCE_CONFIG_OUT / "cosmosis_pipeline_{version}_A_ia.ini" ), config_file_sacc=str( - INFERENCE_TEMPLATES / "cosmosis_pipeline_{version}_A_ia_sacc.ini" + INFERENCE_CONFIG_OUT / "cosmosis_pipeline_{version}_A_ia_sacc.ini" ), params: # SCRATCH = the per-version chain output root the generated inis point at. scratch=lambda w: f"{CHAINS_DIR}/{w.version}", cosmosis_dir=lambda w: _csl_dir(), - template_2pt=str(INFERENCE_TEMPLATES / "cosmosis_pipeline_A_ia.ini"), - template_sacc=str(INFERENCE_TEMPLATES / "cosmosis_pipeline_A_ia_sacc.ini"), threads: 1 resources: mem_mb=8000, @@ -124,7 +135,7 @@ rule inference_prep: # (a) converter 2pt-FITS — pure ξ (A_ia scope; no rho/tau sidecars). sacc_to_twopoint_fits(sacc_io.load(input.sacc), output.fits_file, n_bins=1) - # (b) + (c) the two generated pipeline inis, from the existing templates. + # (b) + (c) the two generated pipeline inis, from the template inputs. # WORKFLOW_SCRIPTS (common.py) is the absolute generic-workflow scripts dir. sys.path.insert(0, WORKFLOW_SCRIPTS) from generate_inference_config import ( @@ -133,7 +144,7 @@ rule inference_prep: ) generate_inference_config( - params.template_2pt, + input.template_2pt, output.config_file_2pt, _substitutions( scratch=params.scratch, @@ -142,7 +153,7 @@ rule inference_prep: ), ) generate_inference_config( - params.template_sacc, + input.template_sacc, output.config_file_sacc, _substitutions( scratch=params.scratch, From 151d997ee0ea74e8c35a1c16a2cc023492c3e20f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 17:24:48 +0200 Subject: [PATCH 32/38] fix(sacc_like): defer cosmosis import so bare module import needs no cosmosis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI (test_imports.py::test_package_module_imports[sp_validation.sacc_like_unions]) failed: the CI image has the science stack but no cosmosis, and the shim imported `from cosmosis.datablock import SectionOptions, option_section` at module level. cosmosis is an optional dependency, so a bare `import sp_validation.sacc_like_unions` must succeed without it (test_imports bare-imports every top-level module). Move the cosmosis import into setup() (call time). Nothing at top level touches cosmosis now — only os/sys/numpy (all in the image). The factory's subclass of the upstream SaccClLikelihood already deferred (setup imports it from csl_dir), so setup() is the single cosmosis entry point. Verified: in the container (numpy present, cosmosis genuinely absent — the CI condition), `import sp_validation.sacc_like_unions` succeeds with setup/execute/ cleanup/ARCMIN_TO_RAD all present, and sacc_like_unions is absent from the import-sweep failures. Full function intact: test_sacc_like.py 8 passed under the venv (cosmosis present). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01EnV8NPWhkxS2SxGVgGSJyt --- src/sp_validation/sacc_like_unions.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/sp_validation/sacc_like_unions.py b/src/sp_validation/sacc_like_unions.py index 5aa610ec..a25837cb 100644 --- a/src/sp_validation/sacc_like_unions.py +++ b/src/sp_validation/sacc_like_unions.py @@ -54,7 +54,12 @@ import sys import numpy as np -from cosmosis.datablock import SectionOptions, option_section + +# cosmosis is an OPTIONAL dependency: this module is a CosmoSIS module file, but +# importing it must succeed without cosmosis installed (the CI image has the +# science stack but no cosmosis, and test_imports.py bare-imports every module). +# So every cosmosis touch is deferred into setup() — nothing at top level imports +# it. numpy is fine at top level (always present). # arcmin → radian: the conversion 2pt_like applies to real-space data and that # sacc_like omits. Applied only to `theta` tags of `real`-category data types. @@ -195,8 +200,12 @@ def setup(options): Mirrors ``GaussianLikelihood.build_module``'s setup: wrap the raw options in ``SectionOptions`` and instantiate the likelihood (whose ``__init__`` calls ``build_data``). The one addition is reading ``csl_dir`` from the module - options to locate and import the upstream class before subclassing it. + options to locate and import the upstream class before subclassing it. The + cosmosis import is deferred to here (call time) so importing this module never + requires cosmosis. """ + from cosmosis.datablock import SectionOptions, option_section + csl_dir = options.get_string(option_section, "csl_dir") sacc_like = _import_upstream_sacc_like(csl_dir) likelihood_class = _make_subclass(sacc_like) From 64b2b5f8247b0f976f80bbab2e23400b3d40a850 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:33:59 +0200 Subject: [PATCH 33/38] =?UTF-8?q?feat:=20SACC=20=E2=86=92=202pt-FITS=20con?= =?UTF-8?q?verter=20+=20OneCovariance=20file-format=20glue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds twopoint_convert (SACC → 2pt-FITS, byte-compatible with cosmosis_fitting) and one_covariance_io (SACC ↔ OneCovariance glue), with their test suites. Rebuilt on feat/sacc-2-sacc-io so the diff is converter-only; adapted to that branch's canonical sacc_io — ξ reads use grid='reporting' (was 'coarse'). Optional-block probing (pseudo-Cℓ, τ) stays on raw s.indices behind get_data_types/use_rho_tau guards, so the new fail-loud sacc_io readers don't trip on absent blocks. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- src/sp_validation/one_covariance_io.py | 256 ++++++++++ .../tests/test_one_covariance_io.py | 274 ++++++++++ .../tests/test_twopoint_convert.py | 472 ++++++++++++++++++ .../tests/test_twopoint_convert_realdata.py | 268 ++++++++++ src/sp_validation/twopoint_convert.py | 368 ++++++++++++++ 5 files changed, 1638 insertions(+) create mode 100644 src/sp_validation/one_covariance_io.py create mode 100644 src/sp_validation/tests/test_one_covariance_io.py create mode 100644 src/sp_validation/tests/test_twopoint_convert.py create mode 100644 src/sp_validation/tests/test_twopoint_convert_realdata.py create mode 100644 src/sp_validation/twopoint_convert.py diff --git a/src/sp_validation/one_covariance_io.py b/src/sp_validation/one_covariance_io.py new file mode 100644 index 00000000..4c0f2ae4 --- /dev/null +++ b/src/sp_validation/one_covariance_io.py @@ -0,0 +1,256 @@ +"""ONE_COVARIANCE_IO. + +:Name: one_covariance_io.py + +:Description: File-format glue between the SACC data-product layout + (:mod:`sp_validation.sacc_io`) and OneCovariance + (https://github.com/rreischke/OneCovariance). Two directions: + + - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the + ``source_i`` NZ tracers of an analysis SACC are written as the + combined whitespace-delimited redshift file OneCovariance reads + (column 0 = z grid, then one ``n(z)`` column per tomographic + bin, no bin edges), and a matching ``[redshift]`` config stanza + is returned via :func:`nz_config_stanza`. + + - **OneCovariance output -> SACC covariance blocks** + (:func:`covariance_blocks`): the flat ``covariance_list_*.dat`` + table OneCovariance emits (one row per element pair) is reshaped + into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the + per-block reshape — and paired with SACC selectors so a caller + can feed them straight to + :func:`sp_validation.sacc_io.assemble_covariance`. + + OneCovariance itself is *not* a dependency: this module only + touches its file formats, verified against the upstream + ``config.ini`` (``rreischke/OneCovariance`` @ main). + + n(z) file format (upstream ``config.ini`` comment, verbatim): + + ``redshift n_1(z) ... n_{N_source}(z)`` + + i.e. a plain whitespace-delimited text file, column 0 the shared + redshift grid and one column per tomographic bin — no ``z_low``/ + ``z_high`` edges (this is the OneCovariance convention, distinct + from the CosmoSIS NZDATA table which *does* carry edges). All + source bins must therefore share one z grid. + + ``[redshift]`` config keys (upstream canonical names): a single + combined file goes in ``zlens_directory`` + ``zlens_file``; + ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where + in each histogram bin the tabulated ``n(z)`` value sits — ``mid`` + for the bin-centred grids the SACC stores. NOTE: the UNIONS + OneCovariance template driven by + ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older + key names ``z_directory``/``zlens_file`` instead; pass + ``dir_key="z_directory"`` to match that template. +""" + +import os + +import numpy as np + +from . import sacc_io +from .statistics import cov_from_one_covariance + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = sacc_io.get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if sacc_io.source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.sacc_io.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``sacc_io.assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_one_covariance_io.py b/src/sp_validation/tests/test_one_covariance_io.py new file mode 100644 index 00000000..9bc7da56 --- /dev/null +++ b/src/sp_validation/tests/test_one_covariance_io.py @@ -0,0 +1,274 @@ +"""Tests for :mod:`sp_validation.one_covariance_io`. + +All synthetic, all fast: the OneCovariance fixtures are built in memory +shaped exactly like its real file I/O — a flat ``covariance_list`` table with +the ``(i, j)`` index rows and the Gaussian / Gauss+non-Gaussian value columns +at the indices ``cov_from_one_covariance`` expects (col 10 / col 9), and the +combined n(z) text file (column 0 = z, one column per bin, no edges). No +cluster paths; OneCovariance is not imported. + +The two pieces: + +- **Piece 1** (n(z) SACC -> OneCovariance input): write the combined n(z) + file from a SACC's ``source_i`` NZ tracers, read it back, assert the z and + n(z) columns round-trip and the config stanza names the file. +- **Piece 2** (OneCovariance output -> SACC covariance blocks): reshape the + flat table to the dense block(s) and prove they feed + ``sacc_io.assemble_covariance`` cleanly (reshaped block -> assemble -> + ``s.covariance.dense`` matches the hand-built matrix). +""" + +import numpy as np +import numpy.testing as npt +import pytest + +from sp_validation import one_covariance_io as ocio +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic OneCovariance-shaped fixtures +# --------------------------------------------------------------------------- # +def _one_cov_table(cov_gauss, cov_all): + """Flatten two n x n matrices into a OneCovariance ``covariance_list`` table. + + Reproduces the real flat output: one row per ``(i, j)`` element pair in + row-major order ``k = i·n + j``, with the Gaussian value in column 10 and + the Gaussian+non-Gaussian value in column 9. Columns 0-8 and the index + columns are filled with self-documenting placeholder values (the reshape + only reads cols 9/10, but a realistic width proves it does not spill). + """ + n = cov_gauss.shape[0] + rows = [] + for i in range(n): + for j in range(n): + row = np.arange(11.0) # placeholder cols 0-8 (+ overwritten 9,10) + row[9] = cov_all[i, j] + row[10] = cov_gauss[i, j] + rows.append(row) + return np.array(rows) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n`` (a valid covariance).""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _nz(seed, n=40): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +# --------------------------------------------------------------------------- # +# Piece 2 — flat covariance_list -> dense block (reshape correctness + teeth) +# --------------------------------------------------------------------------- # +def test_covariance_blocks_reshapes_to_hand_built_matrix(): + """Pin the reshape and prove gaussian vs gauss+ng select different columns. + + WHAT IS PINNED: ``covariance_blocks`` flattens/reshapes the OneCovariance + ``covariance_list`` table into a dense square block matching a hand-built + covariance. It delegates the per-block reshape to + ``statistics.cov_from_one_covariance``, so column 10 (gaussian) and column 9 + (gauss+ng) must recover the two distinct hand-built matrices. + + WHY TEETH: (a) ``gaussian=True`` vs ``False`` must return the two *different* + matrices, proving the column flag is load-bearing; (b) perturbing a single + entry of the flat input must change exactly that entry of the reshaped + block, proving the reshape actually reads the table (not a constant). + """ + cov_gauss = _spd(4, seed=1) + cov_all = _spd(4, seed=2) + table = _one_cov_table(cov_gauss, cov_all) + + selector = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + + [(sel_g, block_g)] = ocio.covariance_blocks(table, selector, gaussian=True) + [(sel_a, block_a)] = ocio.covariance_blocks(table, selector, gaussian=False) + + assert sel_g == selector and sel_a == selector + npt.assert_allclose(block_g, cov_gauss, rtol=1e-12) + npt.assert_allclose(block_a, cov_all, rtol=1e-12) + + # TEETH: gaussian and gauss+ng select different columns -> different blocks. + assert not np.allclose(block_g, block_a) + + # TEETH: a perturbation of one flat-table entry moves exactly that block + # entry (row k = i·n + j, col 10 for gaussian). + perturbed = table.copy() + perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) + [(_, block_p)] = ocio.covariance_blocks(perturbed, selector, gaussian=True) + npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) + block_p[2, 1] = block_g[2, 1] + npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved + + +def test_covariance_blocks_multiblock_form(): + """Prove the tomography-ready multi-block form reshapes each sub-table. + + WHAT IS PINNED: passing ``selectors=None`` and a sequence of + ``(selector, sub_table)`` pairs reshapes each sub-table independently and + returns them paired with their selectors in order — the shape needed to map + a multi-statistic OneCovariance output onto several SACC selectors. + + WHY TEETH: the two sub-tables carry different matrices; if the function + reshaped only the first or mixed them, the second block would not match its + own hand-built matrix. + """ + cov_a, cov_b = _spd(3, seed=3), _spd(2, seed=4) + table_a = _one_cov_table(cov_a, _spd(3, seed=5)) + table_b = _one_cov_table(cov_b, _spd(2, seed=6)) + sel_a = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + sel_b = (sio.XI_MINUS, (sio.source_name(0), sio.source_name(0))) + + blocks = ocio.covariance_blocks( + [(sel_a, table_a), (sel_b, table_b)], None, gaussian=True + ) + + assert [s for s, _ in blocks] == [sel_a, sel_b] + npt.assert_allclose(blocks[0][1], cov_a, rtol=1e-12) + npt.assert_allclose(blocks[1][1], cov_b, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 2 — the reshaped block feeds assemble_covariance cleanly +# --------------------------------------------------------------------------- # +def test_covariance_blocks_feed_assemble_covariance(): + """Round-trip: reshaped block -> assemble_covariance -> dense matches. + + WHAT IS PINNED: the ``(selector, dense)`` pair ``covariance_blocks`` + returns is directly consumable by ``sacc_io.assemble_covariance``: assembled + onto a SACC whose only statistic is one ξ+ block, ``s.covariance.dense`` + must equal the hand-built OneCovariance matrix. This is the end-to-end + contract between the two modules. + + WHY TEETH: the block must tile the data vector exactly; if the reshape + produced the wrong size or the wrong selector, ``assemble_covariance`` would + raise (its contiguity/tiling/size checks), so a clean assemble + matching + dense is a real proof. + """ + theta = np.geomspace(1.0, 100.0, 4) + xip, xim = np.arange(4) * 1e-5, np.arange(4) * 2e-5 + s = sio.new_sacc({0: _nz(0)}) + sio.add_xi(s, (0, 0), theta, xip, xim, grid="reporting") + + # The ξ block spans ξ+ then ξ− for the pair -> 8 points, one contiguous + # block (pair-major, matching sacc_io's canonical order). + cov_gauss = _spd(8, seed=7) + table = _one_cov_table(cov_gauss, _spd(8, seed=8)) + pair = (sio.source_name(0), sio.source_name(0)) + selector = np.concatenate( + [s.indices(sio.XI_PLUS, pair), s.indices(sio.XI_MINUS, pair)] + ) + + blocks = ocio.covariance_blocks(table, selector, gaussian=True) + sio.assemble_covariance(s, blocks) + + npt.assert_allclose(s.covariance.dense, cov_gauss, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 1 — n(z) SACC -> OneCovariance input (round-trip + config stanza) +# --------------------------------------------------------------------------- # +def test_write_nz_roundtrips_and_names_file(tmp_path): + """Round-trip the n(z) file and check the config stanza names it. + + WHAT IS PINNED: ``write_nz`` writes the SACC ``source_i`` NZ tracers as the + OneCovariance combined file (column 0 = z, one column per bin, no z_low/ + z_high edges). ``read_nz`` recovers the z grid and every per-bin n(z) + column, and the returned ``[redshift]`` stanza names the exact file written + (directory + basename) plus ``value_loc_in_lensbin``. + + WHY TEETH: the z grid and each n(z) column must round-trip to the values the + SACC holds (drawn from a seeded RNG); a transposed write or a dropped column + would fail the per-bin comparison. The stanza's directory/file must match + the path actually written. + """ + z, nz0 = _nz(10) + _, nz1 = _nz(11) + s = sio.new_sacc({0: (z, nz0), 1: (z, nz1)}) + + path = tmp_path / "nz_onecov.txt" + stanza = ocio.write_nz(s, path, n_bins=2) + + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + assert nz_read.shape == (len(z), 2) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + npt.assert_allclose(nz_read[:, 1], nz1, rtol=1e-12) + + assert stanza["zlens_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz_onecov.txt" + assert stanza["value_loc_in_lensbin"] == "mid" + + +def test_write_nz_unions_template_dir_key(tmp_path): + """The UNIONS-template ``z_directory`` key is selectable via ``dir_key``. + + WHAT IS PINNED: the upstream OneCovariance key is ``zlens_directory``, but + the UNIONS template (pseudo_cl.py._modify_onecov_config) writes + ``z_directory``. ``dir_key="z_directory"`` produces that variant so the + stanza drops straight into the UNIONS template's ``[redshift]`` section. + """ + z, nz0 = _nz(12) + s = sio.new_sacc({0: (z, nz0)}) + stanza = ocio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") + assert "z_directory" in stanza and "zlens_directory" not in stanza + assert stanza["z_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz.txt" + + +def test_write_nz_fails_on_mismatched_z_grids(tmp_path): + """Fail fast when source bins do not share one redshift grid. + + WHAT IS PINNED: the OneCovariance combined file has a single redshift + column, so all bins must share the z grid. A bin on a different grid must + raise ``ValueError`` at write time, not silently mis-align. + """ + z0, nz0 = _nz(20) + z1_shifted, nz1 = _nz(21) + z1_shifted = z1_shifted + 0.1 # different grid + s = sio.new_sacc({0: (z0, nz0), 1: (z1_shifted, nz1)}) + with pytest.raises(ValueError, match="differs from source bin 0"): + ocio.write_nz(s, tmp_path / "bad.txt", n_bins=2) + + +def test_write_nz_no_header_roundtrips(tmp_path): + """The bare (header=False) file still round-trips numerically. + + WHAT IS PINNED: ``header=False`` writes a purely numeric file (no ``#`` + column-name line); ``read_nz`` recovers the same z grid and n(z) column, so + the header is cosmetic and never load-bearing for the numeric round-trip. + """ + z, nz0 = _nz(40) + s = sio.new_sacc({0: (z, nz0)}) + path = tmp_path / "bare.txt" + ocio.write_nz(s, path, n_bins=1, header=False) + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + + +def test_nz_config_stanza_rejects_bad_value_loc(): + """``value_loc_in_lensbin`` outside {mid,left,right} fails fast. + + WHAT IS PINNED: OneCovariance only accepts ``mid``/``left``/``right`` for + the histogram-bin value location; an invalid value is a config bug and must + raise ``ValueError`` rather than write a stanza OneCovariance will reject. + """ + with pytest.raises(ValueError, match="value_loc_in_lensbin"): + ocio.nz_config_stanza("/dir", "nz.txt", value_loc="center") + + +def test_write_nz_fails_on_missing_bin(tmp_path): + """Fail fast when a requested source bin is absent from the SACC. + + WHAT IS PINNED: requesting more bins than the SACC carries is a real config + bug; ``write_nz`` raises ``ValueError`` naming the missing tracer rather + than writing a short file. + """ + z, nz0 = _nz(30) + s = sio.new_sacc({0: (z, nz0)}) + with pytest.raises(ValueError, match="source_1"): + ocio.write_nz(s, tmp_path / "short.txt", n_bins=2) diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py new file mode 100644 index 00000000..d50e33b9 --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -0,0 +1,472 @@ +"""Byte-compare tests for the SACC -> 2pt-FITS converter. + +The converter (:mod:`sp_validation.twopoint_convert`) must reproduce the CosmoSIS +2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, +so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau +``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check +is *byte* equality, and astropy writes FITS deterministically, so that is what we +assert: build a reference with the current script's own HDU-builder functions on +deterministic synthetic inputs, build a SACC from those same inputs via +:mod:`sp_validation.sacc_io`, convert it, and compare the two files byte for byte. + +Three configurations pin the three product shapes today's ``__main__`` emits: + +1. **plain xi** -- PRIMARY, NZ_SOURCE, COVMAT, XI_PLUS, XI_MINUS. +2. **xi + pseudo-Cl** -- adds COVMAT_CELL and CELL_EE (the harmonic block + ``2pt_like`` reads; the script builds CELL_BB and discards it, so the + converter does too). +3. **xi + rho/tau** -- adds the blocked tau covariance (TAU_0_PLUS / TAU_2_PLUS, + with the tau_0<->tau_2 cross-correlation the truncated CosmoCov tau + covariance carries) and the verbatim RHO_STATS table. + +The rho/tau product needs the rho/tau *sidecar* HDUs: the RHO_STATS table carries +per-mode ``varrho_*`` variances the analysis SACC does not store, so the +converter copies them from the sidecar exactly as today's assembly does. A teeth +test pins that a perturbed input moves the output. + +The reference builder imports ``cosmosis_fitting.py`` by path (it is a script, +not a package module), skipping cleanly if a dependency is missing -- the same +loader pattern as ``test_cosmosis_fitting.py``. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + """Import cosmosis_fitting.py by path; skip cleanly if a dep is missing.""" + if not _SCRIPT.exists(): + pytest.skip(f"cosmosis_fitting.py not found at {_SCRIPT}") + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except ImportError as exc: # pragma: no cover - container has numpy/astropy + pytest.importorskip(getattr(exc, "name", "") or "cosmosis_fitting_dependency") + raise + return module + + +cf = _load_cf() + + +# --- deterministic synthetic inputs ----------------------------------------- +# +# One tomographic bin (today's analysis). N_ANG angular bins on an ascending +# theta grid; N_ELL bandpowers; a 200-point n(z) on a uniform z grid (the DES +# NZDATA table needs a uniform Z_MID axis). The xi covariance is a full +# (2*N_ANG) matrix (xi+/xi- cross-block nonzero, as CosmoCov produces); the tau +# covariance is a full (3*N_ANG) matrix that the assembly truncates to its first +# 2 statistics (tau_0, tau_2). + +N_ANG = 5 +N_ELL = 8 +SOURCE = sacc_io.source_name(0) +PSF = sacc_io.PSF_TRACER + + +def _spd(n, seed): + """A symmetric positive-definite (n, n) matrix, seeded and recognizable.""" + rng = np.random.default_rng(seed) + a = rng.normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _inputs(seed=0): + """Deterministic synthetic statistics + covariances for one bin pair.""" + rng = np.random.default_rng(seed) + theta = np.sort(rng.uniform(1.0, 250.0, N_ANG)) + ell = np.sort(rng.uniform(30.0, 3000.0, N_ELL)) + z = np.linspace(0.0125, 0.4875, 200) + return { + "theta": theta, + "ell": ell, + "z": z, + "nz": np.exp(-((z - 0.25) ** 2) / 0.02), + "xip": rng.uniform(1e-6, 1e-4, N_ANG), + "xim": rng.uniform(1e-6, 1e-4, N_ANG), + "cl_ee": rng.uniform(1e-10, 1e-8, N_ELL), + "cl_bb": rng.uniform(1e-11, 1e-9, N_ELL), + "cl_eb": rng.uniform(-1e-11, 1e-11, N_ELL), + "tau0p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau0m": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2m": rng.uniform(1e-6, 1e-5, N_ANG), + "xi_cov": _spd(2 * N_ANG, seed + 1), + "cl_cov": _spd(N_ELL, seed + 2), + "tau_cov_full": _spd(3 * N_ANG, seed + 3), + } + + +# --- reference sidecar files + HDUs (the current script's own builders) ------ + + +def _rho_sidecar_hdu(theta, seed=7): + """A rho-stats BinTableHDU with the 25-column layout (values + variances).""" + rng = np.random.default_rng(seed) + columns = [fits.Column(name="theta", format="D", array=theta)] + for k in range(6): + for suffix in ("_p", "_m"): + columns.append( + fits.Column( + name=f"rho_{k}{suffix}", + format="D", + array=rng.uniform(-1e-3, 1e-3, len(theta)), + ) + ) + columns.append( + fits.Column( + name=f"varrho_{k}{suffix}", + format="D", + array=rng.uniform(1e-15, 1e-12, len(theta)), + ) + ) + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _tau_sidecar_hdu(theta, tau0p, tau2p): + """A tau-stats BinTableHDU with theta + tau_0_p + tau_2_p columns.""" + columns = [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _reference_fits(tmp_path, inp, *, cl=False, rho_tau=False): + """Build the reference 2pt-FITS with cosmosis_fitting.py's own functions. + + Reproduces the exact ``__main__`` HDU list for the requested configuration, + which is the assembly the converter must match byte for byte. + """ + nz_txt = tmp_path / "nz.txt" + np.savetxt(nz_txt, np.column_stack([inp["z"], inp["nz"]])) + cov_txt = tmp_path / "cov_xi.txt" + np.savetxt(cov_txt, inp["xi_cov"]) + + nz_hdu = cf.nz_to_fits(str(nz_txt)) + xip_hdu = cf._create_2pt_hdu(inp["xip"], inp["theta"], "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(inp["xim"], inp["theta"], "XI_MINUS", "G-R", "G-R") + + hdu_list = [fits.PrimaryHDU(), nz_hdu] + + if rho_tau: + tau_cov_npy = tmp_path / "cov_tau.npy" + np.save(tau_cov_npy, inp["tau_cov_full"]) + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=str(tau_cov_npy)) + else: + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=None) + hdu_list.append(cov_hdu) + + if cl: + cl_block = np.zeros((5, N_ELL)) + cl_block[0], cl_block[1], cl_block[4] = inp["ell"], inp["cl_ee"], inp["cl_bb"] + cl_npy = tmp_path / "cl.npy" + np.save(cl_npy, cl_block) + cl_cov_npy = tmp_path / "cl_cov.npy" + np.save(cl_cov_npy, inp["cl_cov"]) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(cl_npy)) + cl_ee_hdu, _cl_bb_hdu = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(cl_cov_npy), cov_hdu="COVAR_FULL") + hdu_list.append(cov_cl_hdu) + + hdu_list.extend([xip_hdu, xim_hdu]) + if cl: + hdu_list.append(cl_ee_hdu) + + if rho_tau: + rho_path = tmp_path / "rho.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"]), + ] + ).writeto(tau_path, overwrite=True) + rho_hdu = cf.rho_to_fits(str(rho_path), theta=inp["theta"]) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tau_path), theta=inp["theta"]) + hdu_list.extend([tau0_hdu, tau2_hdu, rho_hdu]) + + out = tmp_path / "reference.fits" + fits.HDUList(hdu_list).writeto(out, overwrite=True) + return out + + +# --- the SACC each configuration is built from ------------------------------ + + +def _sacc(inp, *, cl=False, rho_tau=False): + """Build the analysis SACC the converter reads, matching ``_inputs``. + + The covariance is laid out to match the reference exactly: the xi block is + the full (2*N_ANG) matrix; the Cl block is the EE bandpower covariance; the + tau blocks carry the tau_0<->tau_2 cross-correlation from the truncated + CosmoCov tau covariance. Blocks not consumed by the 2pt-FITS (Cl BB/EB, tau + minus) get an identity block so ``add_covariance`` sees a full matrix. + """ + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_xi(s, (0, 0), inp["theta"], inp["xip"], inp["xim"], grid="reporting") + if cl: + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + if rho_tau: + sacc_io.add_tau(s, (0, 0), 0, inp["theta"], inp["tau0p"], inp["tau0m"]) + sacc_io.add_tau(s, (0, 0), 2, inp["theta"], inp["tau2p"], inp["tau2m"]) + + n = len(s.mean) + full = np.zeros((n, n)) + ip = s.indices(sacc_io.XI_PLUS, (SOURCE, SOURCE)) + im = s.indices(sacc_io.XI_MINUS, (SOURCE, SOURCE)) + xi_all = np.concatenate([ip, im]) + full[np.ix_(xi_all, xi_all)] = inp["xi_cov"] + + if cl: + iee = s.indices(sacc_io.CL_EE, (SOURCE, SOURCE)) + full[np.ix_(iee, iee)] = inp["cl_cov"] + for dtype in (sacc_io.CL_BB, sacc_io.CL_EB): + idx = s.indices(dtype, (SOURCE, SOURCE)) + full[np.ix_(idx, idx)] = np.eye(N_ELL) + if rho_tau: + t0p = s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)) + t2p = s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)) + tau_pp = np.concatenate([t0p, t2p]) + # The joint [tau_0+; tau_2+] block is the truncated CosmoCov tau + # covariance -- cross-correlation kept, matching covdat_to_fits. + full[np.ix_(tau_pp, tau_pp)] = inp["tau_cov_full"][: 2 * N_ANG, : 2 * N_ANG] + for dtype in (sacc_io.TAU_MINUS.format(k=0), sacc_io.TAU_MINUS.format(k=2)): + idx = s.indices(dtype, (SOURCE, PSF)) + full[np.ix_(idx, idx)] = np.eye(N_ANG) + s.add_covariance(full) + return s + + +def _sidecar_hdus(tmp_path, inp): + """Return the (rho_hdu, tau_hdu) sidecar input HDUs for the rho/tau product.""" + rho_path = tmp_path / "rho_in.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau_in.fits" + fits.HDUList( + [fits.PrimaryHDU(), _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"])] + ).writeto(tau_path, overwrite=True) + with fits.open(rho_path) as r, fits.open(tau_path) as t: + return r[1].copy(), t[1].copy() + + +# ============================================================================= +# Byte-compare: the three product shapes +# ============================================================================= + + +def test_plain_xi_byte_equal(tmp_path): + """Plain-xi product: converter matches cosmosis_fitting.py byte for byte.""" + inp = _inputs(seed=0) + reference = _reference_fits(tmp_path, inp) + s = _sacc(inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_cl_byte_equal(tmp_path): + """xi + pseudo-Cl product: COVMAT_CELL + CELL_EE reproduced byte for byte.""" + inp = _inputs(seed=10) + reference = _reference_fits(tmp_path, inp, cl=True) + s = _sacc(inp, cl=True) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_rho_tau_byte_equal(tmp_path): + """xi + rho/tau product: blocked tau covariance + verbatim RHO_STATS match.""" + inp = _inputs(seed=20) + reference = _reference_fits(tmp_path, inp, rho_tau=True) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + assert out.read_bytes() == reference.read_bytes() + + +# ============================================================================= +# Structural + teeth checks +# ============================================================================= + + +def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): + """The tau covariance block couples tau_0 and tau_2 (not block-diagonal). + + covdat_to_fits truncates the 3-statistic CosmoCov tau covariance to its + first 2 blocks and lays it in as ONE contiguous block, so tau_0<->tau_2 + cross-terms survive. A block-diagonal shortcut would zero them; pin that + the converter keeps them. + """ + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(out) as hdul: + cov = hdul["COVMAT"].data + # tau_0 block rows 2*N_ANG..3*N_ANG, tau_2 block 3*N_ANG..4*N_ANG. + cross = cov[2 * N_ANG : 3 * N_ANG, 3 * N_ANG : 4 * N_ANG] + expected = inp["tau_cov_full"][:N_ANG, N_ANG : 2 * N_ANG] + assert np.array_equal(cross, expected) + assert np.any(cross != 0.0) + + +def test_perturbed_xi_changes_output(tmp_path): + """Teeth: a changed xi+ input must change the converted data vector.""" + inp = _inputs(seed=0) + s = _sacc(inp) + out = tmp_path / "base.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + with fits.open(out) as hdul: + base_xip = hdul["XI_PLUS"].data["VALUE"].copy() + + inp2 = _inputs(seed=0) + inp2["xip"] = inp2["xip"] + 1.0 + s2 = _sacc(inp2) + out2 = tmp_path / "perturbed.fits" + twopoint_convert.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + with fits.open(out2) as hdul: + new_xip = hdul["XI_PLUS"].data["VALUE"] + + assert not np.array_equal(base_xip, new_xip) + assert np.array_equal(new_xip, inp2["xip"]) + + +def test_rho_tau_sidecars_required_together(tmp_path): + """Supplying only one of the rho/tau sidecars is a loud error.""" + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) + with pytest.raises(ValueError, match="together"): + twopoint_convert.sacc_to_twopoint_fits( + s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 + ) + + +# ============================================================================= +# Fail-fast guards and permutation teeth (adversarial-review hardening) +# ============================================================================= + + +def test_tomographic_sacc_raises(tmp_path): + """A multi-bin SACC fails fast instead of silently truncating to (0, 0). + + Review finding (HIGH): ``n_bins`` alone drove the NZDATA column count while + the data vector and covariance were read from bin ``(0, 0)`` only, so a + 2-bin SACC + ``n_bins=2`` emitted a plausible-looking FITS carrying 1/3 of + the data. Both the ``n_bins`` and the tracer-pair mismatch must raise. + """ + inp = _inputs(seed=30) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"]), 1: (inp["z"], inp["nz"])}) + for pair in [(0, 0), (0, 1), (1, 1)]: + sacc_io.add_xi(s, pair, inp["theta"], inp["xip"], inp["xim"], grid="reporting") + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + assert not (tmp_path / "x.fits").exists() + + +def test_sacc_without_xi_raises(tmp_path): + """A SACC with no ξ± points raises instead of writing an empty data vector.""" + inp = _inputs(seed=31) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="nothing to convert"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + assert not (tmp_path / "x.fits").exists() + + +def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): + """Every COVMAT/COVMAT_CELL entry is the exact ``np.ix_`` gather of the SACC + covariance, pinned with a (row, col)-encoded matrix. + + Review finding (MEDIUM): for a single bin pair the ξ gather happens to be + the identity permutation, so the byte-compares alone could pass with a + transposed or block-swapped gather. Encoding ``C[i, j] = i*n + j`` (asymmetric, + every entry unique) makes any transposition, offset, or wrong block produce + detectably wrong values; the τ gather is genuinely non-identity (τ_0− sits + between τ_0+ and τ_2+ in insertion order). Expected layout per + ``covdat_to_fits``: block_diag(ξ type-major gather, joint [τ_0+; τ_2+] + gather), with COVMAT_CELL the CELL_EE gather in its own HDU. + """ + inp = _inputs(seed=32) + s = _sacc(inp, cl=True, rho_tau=True) + n = len(s.mean) + encoded = np.arange(n * n, dtype=float).reshape(n, n) + s.add_covariance(encoded, overwrite=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + + out = tmp_path / "encoded.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + pair = (SOURCE, SOURCE) + xi_idx = np.concatenate( + [s.indices(sacc_io.XI_PLUS, pair), s.indices(sacc_io.XI_MINUS, pair)] + ) + tau_idx = np.concatenate( + [ + s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), + s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), + ] + ) + expected = twopoint_convert._block_diag( + encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] + ) + cell_idx = s.indices(sacc_io.CL_EE, pair) + + with fits.open(out) as hdul: + np.testing.assert_array_equal(hdul["COVMAT"].data, expected) + np.testing.assert_array_equal( + hdul["COVMAT_CELL"].data, encoded[np.ix_(cell_idx, cell_idx)] + ) diff --git a/src/sp_validation/tests/test_twopoint_convert_realdata.py b/src/sp_validation/tests/test_twopoint_convert_realdata.py new file mode 100644 index 00000000..1dcf920f --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert_realdata.py @@ -0,0 +1,268 @@ +"""Candide-local byte-compare of the converter against real 2pt-FITS products. + +Skipped unless a real product exists on disk (candide only; never committed). +It closes the loop end to end on real data: take a real CosmoSIS 2pt-FITS, build +an analysis SACC from its own contents, convert that SACC back to a 2pt-FITS, and +byte-compare. + +The reference is *not* the on-disk file directly. The committed on-disk products +were written by an older ``cosmosis_fitting.py`` (they carry a CELL_BB HDU and +order COVMAT before NZ_SOURCE); the converter reproduces the *current* script, +which the PR-4 migration will use to regenerate them. So the meaningful contract +-- "the converter equals the current writer" -- is tested by running the current +``cosmosis_fitting.py`` builders on the same real contents and byte-comparing the +converter against that. For transparency the test also records the direct diff +against the stale on-disk file, and asserts only that it differs *by whole HDUs* +(the extra CELL_BB), not in any shared block -- i.e. the drift is purely the +known HDU-set change, with no silent data corruption. + +Observed on 2026-07-10 for ``SP_v1.4.6_leak_corr`` and a ``glass_mock`` sibling: +converter == current-script byte for byte; converter vs on-disk differs only by +the CELL_BB HDU. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") +_REAL_FILES = { + "SP_v1.4.6_leak_corr": _DATA + / "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" + / "cosmosis_SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits", + "glass_mock_00001": _DATA / "glass_mock_00001" / "cosmosis_glass_mock_00001.fits", +} + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sacc_from_2pt_fits(hdul): + """Build an analysis SACC from a real CosmoSIS 2pt-FITS's own contents. + + Reads the NZ, ξ±, pseudo-Cℓ (EE/BB) and blocked covariance back out of the + product and lays them into the standard SACC layout — the inverse direction + the converter then undoes. τ± minus and Cℓ EB are not in the 2pt-FITS, so + they are stored as zeros with identity covariance sub-blocks (the converter + consumes only the ``+``/EE parts). + """ + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + n_ell = len(ell) + + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + xi_cov = covmat[: 2 * n, : 2 * n] + tau_joint = covmat[2 * n : 4 * n, 2 * n : 4 * n] + + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi(s, (0, 0), theta, xip, xim, grid="reporting") + sacc_io.add_pseudo_cl( + s, + (0, 0), + ell, + cl_ee, + cl_bb, + np.zeros(n_ell), + window_ells=np.arange(2, 102), + window_weights=np.ones((100, n_ell)), + ) + sacc_io.add_tau(s, (0, 0), 0, theta, tau0p, np.zeros(n)) + sacc_io.add_tau(s, (0, 0), 2, theta, tau2p, np.zeros(n)) + + source, psf = sacc_io.source_name(0), sacc_io.PSF_TRACER + idx = { + "xi": np.concatenate( + [ + s.indices(sacc_io.XI_PLUS, (source, source)), + s.indices(sacc_io.XI_MINUS, (source, source)), + ] + ), + "ee": s.indices(sacc_io.CL_EE, (source, source)), + "bb": s.indices(sacc_io.CL_BB, (source, source)), + "eb": s.indices(sacc_io.CL_EB, (source, source)), + "t0p": s.indices(sacc_io.TAU_PLUS.format(k=0), (source, psf)), + "t0m": s.indices(sacc_io.TAU_MINUS.format(k=0), (source, psf)), + "t2p": s.indices(sacc_io.TAU_PLUS.format(k=2), (source, psf)), + "t2m": s.indices(sacc_io.TAU_MINUS.format(k=2), (source, psf)), + } + full = np.zeros((len(s.mean), len(s.mean))) + full[np.ix_(idx["xi"], idx["xi"])] = xi_cov + full[np.ix_(idx["ee"], idx["ee"])] = covmat_cell + tau_pp = np.concatenate([idx["t0p"], idx["t2p"]]) + full[np.ix_(tau_pp, tau_pp)] = tau_joint + for key in ("bb", "eb"): + full[np.ix_(idx[key], idx[key])] = np.eye(n_ell) + for key in ("t0m", "t2m"): + full[np.ix_(idx[key], idx[key])] = np.eye(n) + s.add_covariance(full) + + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + return s, hdul["RHO_STATS"].copy(), tau_sidecar + + +def _current_script_reference(cf, hdul, tmp_path): + """Reference: run the current cosmosis_fitting.py on the real file's contents.""" + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + np.savetxt(tmp_path / "nz.txt", np.column_stack([z, nz])) + np.savetxt(tmp_path / "cov.txt", covmat[: 2 * n, : 2 * n]) + tau_cov = np.zeros((3 * n, 3 * n)) + tau_cov[: 2 * n, : 2 * n] = covmat[2 * n : 4 * n, 2 * n : 4 * n] + np.save(tmp_path / "cov_tau.npy", tau_cov) + cl_block = np.zeros((5, len(ell))) + cl_block[0], cl_block[1], cl_block[4] = ell, cl_ee, cl_bb + np.save(tmp_path / "cl.npy", cl_block) + np.save(tmp_path / "cl_cov.npy", covmat_cell) + + nz_hdu = cf.nz_to_fits(str(tmp_path / "nz.txt")) + xip_hdu = cf._create_2pt_hdu(xip, theta, "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(xim, theta, "XI_MINUS", "G-R", "G-R") + cov_hdu = cf.covdat_to_fits( + str(tmp_path / "cov.txt"), filename_cov_tau=str(tmp_path / "cov_tau.npy") + ) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(tmp_path / "cl.npy")) + cl_ee_hdu, _ = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(tmp_path / "cl_cov.npy"), cov_hdu="COVAR_FULL") + + fits.HDUList([fits.PrimaryHDU(), hdul["RHO_STATS"].copy()]).writeto( + tmp_path / "rho.fits", overwrite=True + ) + rho_hdu = cf.rho_to_fits(str(tmp_path / "rho.fits"), theta=theta) + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + fits.HDUList([fits.PrimaryHDU(), tau_sidecar]).writeto( + tmp_path / "tau.fits", overwrite=True + ) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tmp_path / "tau.fits"), theta=theta) + + out = tmp_path / "reference.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + nz_hdu, + cov_hdu, + cov_cl_hdu, + xip_hdu, + xim_hdu, + cl_ee_hdu, + tau0_hdu, + tau2_hdu, + rho_hdu, + ] + ).writeto(out, overwrite=True) + return out + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_roundtrip_byte_equal(label, tmp_path): + """Converter reproduces the current writer byte for byte on a real product.""" + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + cf = _load_cf() + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + reference = _current_script_reference(cf, hdul, tmp_path) + + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + # The contract: converter == current cosmosis_fitting.py, byte for byte. + assert converted.read_bytes() == reference.read_bytes() + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): + """The stale on-disk file differs from the converter *only* by the CELL_BB HDU. + + Documents (and guards) the known script-version drift: the on-disk products + were written before CELL_BB was dropped from the assembly, so they carry one + extra HDU. Every HDU the two share carries the same data to floating-point + precision — the drift is a whole-HDU addition, never a silent change to a + shared block. (Bin-edge columns differ by ~1e-17 float noise: the stale file + stored a clean ``Z_LOW=0``, while ``z_mid - step/2`` rounds to ``-1.7e-18``; + both are the same number, so the shared-data check is ``allclose``, not + bitwise — bitwise equality is asserted against the *current* writer above.) + """ + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + ondisk_names = [h.name for h in hdul] + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(converted) as conv: + conv_names = [h.name for h in conv] + # The only HDU the on-disk file has that the converter does not. + assert set(ondisk_names) - set(conv_names) == {"CELL_BB"} + # Every shared table HDU carries the same data to float precision. + for name in conv_names: + if name == "PRIMARY": + continue + a, b = hdul[name].data, conv[name].data + if hasattr(a, "names"): + assert all(np.allclose(a[c], b[c]) for c in a.names), name + else: + assert np.allclose(a, b), name diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py new file mode 100644 index 00000000..fc01ac9d --- /dev/null +++ b/src/sp_validation/twopoint_convert.py @@ -0,0 +1,368 @@ +"""TWOPOINT_CONVERT. + +:Name: twopoint_convert.py + +:Description: Convert an analysis SACC file into the "2pt FITS" that CosmoSIS's + ``2pt_like`` (and Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) + reads. The output reproduces today's hand-assembled product from + ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and + byte-for-byte: an NZDATA table, XI_PLUS / XI_MINUS 2pt tables, + optional CELL_EE / CELL_BB pseudo-Cℓ tables, the blocked COVMAT + (with ``STRT_i`` block-offset headers) and separate COVMAT_CELL, + and — when the SACC carries them — the TAU_{0,2}_PLUS 2pt tables + and the RHO_STATS table. + + The converter is the *inverse* of the SACC writers in + :mod:`sp_validation.sacc_io`: it reads statistics back through + those readers and lays them into the DES ``twopoint`` FITS + convention. SACC's canonical order is pair-major (per pair + ``[ξ+; ξ−]``); the 2pt-FITS layout is type-major (all ξ+, then all + ξ−), so the data-vector and its covariance are permuted here via + ``s.indices`` rather than assuming any global order. + + Scope note (single-bin today, tomography-ready): the assembly this + mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ + ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly. + A tomographic 2pt-FITS layout (multiple bin pairs, per-pair + BIN1/BIN2, one NZ column per bin) is a later extension; it is not + what today's CosmoSIS pipeline consumes, so it is out of scope for + the byte-compatible converter. + + Rho/tau caveat: the SACC layout stores ρ±/τ± *values* only, while + the 2pt-FITS RHO_STATS table also carries the per-mode *variances* + (``varrho_*``) that Sacha's fork's covariance path reads. Those + variances are not recoverable from the analysis SACC. The + converter therefore writes RHO_STATS / TAU HDUs only when a + ``rho_stats``/``tau_stats`` sidecar FITS is supplied (the same + file today's assembly copies verbatim); it never fabricates + variances. ξ±, Cℓ, n(z) and the covariance — the data vector + CosmoSIS fits — are fully reconstructed from SACC alone. +""" + +import numpy as np +from astropy.io import fits + +from . import sacc_io + +# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the +# extension name — copied from cosmosis_fitting.py so the headers match card +# for card. +_QUANT = { + "XI_PLUS": ("G+R", "G+R"), + "XI_MINUS": ("G-R", "G-R"), + "CELL_EE": ("GEF", "GEF"), + "CELL_BB": ("GBF", "GBF"), + "TAU_0_PLUS": ("G+R", "P+R"), + "TAU_2_PLUS": ("G+R", "SR+R"), +} + + +def _twopoint_hdu(name, values, ang, *, ang_unit=None): + """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). + + Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: + same column order and formats, the ``2PTDATA`` marker, the QUANT pair for + ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG + column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). + """ + nbins = len(values) + angbin = np.arange(1, nbins + 1) + columns = [ + fits.Column(name="BIN1", format="K", array=np.ones(nbins)), + fits.Column(name="BIN2", format="K", array=np.ones(nbins)), + fits.Column(name="ANGBIN", format="K", array=angbin), + fits.Column(name="VALUE", format="D", array=values), + fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), + ] + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) + quant1, quant2 = _QUANT[name] + for key, value in { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + }.items(): + hdu.header[key] = value + return hdu + + +def _nz_hdu(s, n_bins): + """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. + + Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` + grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` + column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins + are required to share the ``z`` grid — the single ``Z_MID`` axis of the + DES NZDATA table. + """ + z_mid, nz0 = sacc_io.get_nz(s, 0) + z_mid = np.asarray(z_mid, dtype=float) + step = z_mid[1] - z_mid[0] + z_low = z_mid - step / 2 + z_high = z_mid + step / 2 + + columns = [ + fits.Column(name="Z_LOW", format="D", array=z_low), + fits.Column(name="Z_MID", format="D", array=z_mid), + fits.Column(name="Z_HIGH", format="D", array=z_high), + ] + for i in range(n_bins): + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the DES " + "NZDATA table requires one shared Z_MID axis" + ) + columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) + + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") + for key, value in { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": n_bins, + "NZ": len(z_low), + }.items(): + hdu.header[key] = value + return hdu + + +def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): + """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. + + Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for + card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed + and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` + CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME + card is created early, with astropy's standard comment) before re-stamping. + ``name_in_ctor`` selects the second form so the card order matches exactly. + """ + matrix = np.asarray(matrix, dtype=np.float64) + if matrix.shape[0] != matrix.shape[1]: + raise ValueError(f"covariance must be square; got shape {matrix.shape}") + hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) + hdu.header["COVDATA"] = "True" + hdu.header["EXTNAME"] = extname + for i, (name, start) in enumerate(zip(block_names, block_starts)): + hdu.header[f"NAME_{i}"] = name + hdu.header[f"STRT_{i}"] = int(start) + return hdu + + +def _type_major_xi(s, bins): + """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. + + ``sacc_io.get_xi`` already returns each statistic in insertion (= ascending + θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two + arrays it hands back, so no further permutation is needed for a single pair. + """ + return sacc_io.get_xi(s, bins, grid="reporting") + + +def _require_single_bin(s, n_bins): + """Fail fast unless the SACC is a valid single-bin ξ product. + + The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads + (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip + through silently — ``n_bins`` alone drives the NZDATA column count while the + ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a + ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. + Guards both the empty-ξ case and the single-bin contract; tomographic + emission lands with the tomographic round. + """ + pairs = s.get_tracer_combinations(sacc_io.XI_PLUS) + if not pairs: + raise ValueError( + f"SACC has no {sacc_io.XI_PLUS} points — nothing to convert; the " + "2pt-FITS data vector is built from the ξ± statistics" + ) + expected = (sacc_io.source_name(0), sacc_io.source_name(0)) + if n_bins != 1 or set(pairs) != {expected}: + raise ValueError( + f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " + f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " + "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " + "bin) lands with the tomographic round." + ) + + +def sacc_to_twopoint_fits( + s, + path, + *, + rho_stats_hdu=None, + tau_stats_hdu=None, + n_bins=1, +): + """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. + + The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product + for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then + (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, + and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, + RHO_STATS. The data vector and its covariance are laid out type-major + (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` + convention CosmoSIS reads. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the + ρ/τ product — the τ data points; see ``rho_stats_hdu``). + path : str + Output FITS path (overwritten). + rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional + The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's + assembly does. Required together to write the ρ/τ product; the SACC + alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When + omitted, a pure ξ (± Cℓ) product is written. + n_bins : int, optional + Number of source tomographic bins. Must be ``1``: this converter emits + the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic + emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) + lands with the tomographic round; the converter fails fast on anything + else rather than silently truncating to bin ``(0, 0)``. + + Returns + ------- + astropy.io.fits.HDUList + The assembled list, also written to ``path``. + + Raises + ------ + ValueError + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + pairs are anything other than exactly ``{(source_0, source_0)}`` (the + single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. + """ + if (rho_stats_hdu is None) != (tau_stats_hdu is None): + raise ValueError( + "rho_stats_hdu and tau_stats_hdu must be supplied together " + "(the ρ/τ product needs both, or neither for a pure-ξ product)" + ) + _require_single_bin(s, n_bins) + use_rho_tau = rho_stats_hdu is not None + bins = (0, 0) + + nz_hdu = _nz_hdu(s, n_bins) + theta, xip, xim = _type_major_xi(s, bins) + xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") + xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") + + cell_hdu, cov_cell_hdu = _build_cell(s, bins) + + cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) + + tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) + + # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, + # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. + hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] + if cov_cell_hdu is not None: + hdu_list.append(cov_cell_hdu) + hdu_list.extend([xip_hdu, xim_hdu]) + if cell_hdu is not None: + hdu_list.append(cell_hdu) + if use_rho_tau: + hdu_list.extend([*tau_hdus, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(path, overwrite=True) + return hdul + + +def _build_cell(s, bins): + """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. + + Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is + emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's + assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). + The SACC still carries EE/BB/EB with bandpower windows for the B-mode + null-test path; this converter surfaces only the block CosmoSIS reads. The + CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL + ImageHDU, matching today's product. + """ + if sacc_io.CL_EE not in s.get_data_types(): + return None, None + + ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) + cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) + cell_idx = s.indices(sacc_io.CL_EE, sacc_io._pair(bins)) + cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] + cov_cell_hdu = _cov_hdu( + cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True + ) + return cell_hdu, cov_cell_hdu + + +def _build_covmat(s, bins, *, use_rho_tau): + """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). + + The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for + the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under + ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally + with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. + """ + pair = sacc_io._pair(bins) + idx_p = s.indices(sacc_io.XI_PLUS, pair) + idx_m = s.indices(sacc_io.XI_MINUS, pair) + n_theta = len(idx_p) + xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation + xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + names = ["XI_PLUS", "XI_MINUS"] + starts = [0, n_theta] + matrix = xi_cov + + if use_rho_tau: + # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the + # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in + # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the + # SACC those two selections are not adjacent (τ_0− sits between them), so + # gather both index sets and extract the joint sub-block, ξ↔τ zero. + tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) + idx_tau0 = s.indices(sacc_io.TAU_PLUS.format(k=0), tau_pair) + idx_tau2 = s.indices(sacc_io.TAU_PLUS.format(k=2), tau_pair) + tau_idx = np.concatenate([idx_tau0, idx_tau2]) + tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] + matrix = _block_diag(matrix, tau_cov) + names += ["TAU_0_PLUS", "TAU_2_PLUS"] + starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] + + return _cov_hdu(matrix, names, starts) + + +def _block_diag(*blocks): + """Stack square blocks block-diagonally with zero cross-blocks.""" + sizes = [b.shape[0] for b in blocks] + n = sum(sizes) + out = np.zeros((n, n)) + start = 0 + for b in blocks: + out[start : start + b.shape[0], start : start + b.shape[0]] = b + start += b.shape[0] + return out + + +def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): + """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. + + Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` + columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied + verbatim from the sidecar with its θ column forced onto the ξ grid. The + ``varrho_*`` columns ride along in the copy — they are why the sidecar is + required (the SACC cannot supply them). + """ + if not use_rho_tau: + return (), None + + tau = tau_stats_hdu.data + tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") + tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") + + rho_hdu = rho_stats_hdu.copy() + rho_hdu.name = "RHO_STATS" + rho_hdu.data = rho_hdu.data.copy() + rho_hdu.data["theta"] = theta + return (tau0_hdu, tau2_hdu), rho_hdu From b007ed12995af725f6531198604c5df5fab98984 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 18 Jul 2026 17:31:27 +0200 Subject: [PATCH 34/38] twopoint_convert: filter covariance selections by grid tag; guard empty selections Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk --- .../tests/test_twopoint_convert.py | 115 +++++++++++++++++- src/sp_validation/twopoint_convert.py | 14 ++- 2 files changed, 120 insertions(+), 9 deletions(-) diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py index d50e33b9..59e0256d 100644 --- a/src/sp_validation/tests/test_twopoint_convert.py +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -367,6 +367,98 @@ def test_perturbed_xi_changes_output(tmp_path): assert np.array_equal(new_xip, inp2["xip"]) +def test_integration_grid_points_ignored(tmp_path): + """Extra xi/tau/Cl points tagged grid='integration' must not leak into the + converted output. + + Bug report (HIGH): ``_build_covmat`` selected xi+/xi-/tau covariance + indices with raw ``s.indices(dtype, pair)`` -- no ``grid`` filter -- while + the corresponding data-vector HDUs are built via ``sacc_io.get_xi(..., + grid='reporting')``. A real SACC carrying both 'reporting' and + 'integration' grid points under the same data type + tracer pair (e.g. the + fine COSEBIs/pure-EB integration input) would inflate/desync the + covariance relative to the data vector. Pin that the converted file is + byte-identical whether or not the integration-grid points are present. + """ + inp = _inputs(seed=40) + s_plain = _sacc(inp, cl=True, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out_plain = tmp_path / "plain.fits" + twopoint_convert.sacc_to_twopoint_fits( + s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + # Build an augmented SACC: same reporting-grid points, plus a full extra + # set of integration-grid xi/tau/Cl points on a distinct angular grid (so + # they cannot coincide with the reporting points), with the covariance + # sized to cover both blocks. + s_aug = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_xi( + s_aug, (0, 0), inp["theta"], inp["xip"], inp["xim"], grid="reporting" + ) + sacc_io.add_pseudo_cl( + s_aug, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + grid="reporting", + ) + sacc_io.add_tau(s_aug, (0, 0), 0, inp["theta"], inp["tau0p"], inp["tau0m"]) + sacc_io.add_tau(s_aug, (0, 0), 2, inp["theta"], inp["tau2p"], inp["tau2m"]) + + theta_int = inp["theta"] + 1000.0 # disjoint grid, never collides + xip_int = np.random.default_rng(41).uniform(1e-6, 1e-4, N_ANG) + xim_int = np.random.default_rng(42).uniform(1e-6, 1e-4, N_ANG) + sacc_io.add_xi(s_aug, (0, 0), theta_int, xip_int, xim_int, grid="integration") + + n = len(s_aug.mean) + full = np.zeros((n, n)) + ip = sacc_io._indices(s_aug, sacc_io.XI_PLUS, (SOURCE, SOURCE), grid="reporting") + im = sacc_io._indices(s_aug, sacc_io.XI_MINUS, (SOURCE, SOURCE), grid="reporting") + xi_all = np.concatenate([ip, im]) + full[np.ix_(xi_all, xi_all)] = inp["xi_cov"] + + iee = sacc_io._indices(s_aug, sacc_io.CL_EE, (SOURCE, SOURCE), grid="reporting") + full[np.ix_(iee, iee)] = inp["cl_cov"] + for dtype in (sacc_io.CL_BB, sacc_io.CL_EB): + idx = sacc_io._indices(s_aug, dtype, (SOURCE, SOURCE), grid="reporting") + full[np.ix_(idx, idx)] = np.eye(N_ELL) + + t0p = sacc_io._indices( + s_aug, sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF), grid="reporting" + ) + t2p = sacc_io._indices( + s_aug, sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF), grid="reporting" + ) + tau_pp = np.concatenate([t0p, t2p]) + full[np.ix_(tau_pp, tau_pp)] = inp["tau_cov_full"][: 2 * N_ANG, : 2 * N_ANG] + for dtype in (sacc_io.TAU_MINUS.format(k=0), sacc_io.TAU_MINUS.format(k=2)): + idx = sacc_io._indices(s_aug, dtype, (SOURCE, PSF), grid="reporting") + full[np.ix_(idx, idx)] = np.eye(N_ANG) + + ip_int = sacc_io._indices( + s_aug, sacc_io.XI_PLUS, (SOURCE, SOURCE), grid="integration" + ) + im_int = sacc_io._indices( + s_aug, sacc_io.XI_MINUS, (SOURCE, SOURCE), grid="integration" + ) + xi_int_all = np.concatenate([ip_int, im_int]) + full[np.ix_(xi_int_all, xi_int_all)] = _spd(2 * N_ANG, 43) + + s_aug.add_covariance(full) + + out_aug = tmp_path / "aug.fits" + twopoint_convert.sacc_to_twopoint_fits( + s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + assert out_aug.read_bytes() == out_plain.read_bytes() + + def test_rho_tau_sidecars_required_together(tmp_path): """Supplying only one of the rho/tau sidecars is a loud error.""" inp = _inputs(seed=20) @@ -450,20 +542,35 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) + # Expected indices are computed via a code path independent of the shared + # `sacc_io._indices` helper the converter uses: direct boolean filtering on + # each point's data_type/tracers/grid tag, so this test does not + # self-confirm against the converter's own selection logic. + def _mask_idx(dtype, tracers): + return np.array( + [ + i + for i, p in enumerate(s.data) + if p.data_type == dtype + and p.tracers == tracers + and p.tags.get("grid") == "reporting" + ] + ) + pair = (SOURCE, SOURCE) xi_idx = np.concatenate( - [s.indices(sacc_io.XI_PLUS, pair), s.indices(sacc_io.XI_MINUS, pair)] + [_mask_idx(sacc_io.XI_PLUS, pair), _mask_idx(sacc_io.XI_MINUS, pair)] ) tau_idx = np.concatenate( [ - s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), - s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), + _mask_idx(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), + _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) expected = twopoint_convert._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) - cell_idx = s.indices(sacc_io.CL_EE, pair) + cell_idx = _mask_idx(sacc_io.CL_EE, pair) with fits.open(out) as hdul: np.testing.assert_array_equal(hdul["COVMAT"].data, expected) diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py index fc01ac9d..5e3c3156 100644 --- a/src/sp_validation/twopoint_convert.py +++ b/src/sp_validation/twopoint_convert.py @@ -288,7 +288,7 @@ def _build_cell(s, bins): ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) - cell_idx = s.indices(sacc_io.CL_EE, sacc_io._pair(bins)) + cell_idx = sacc_io._indices(s, sacc_io.CL_EE, sacc_io._pair(bins), grid="reporting") cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] cov_cell_hdu = _cov_hdu( cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True @@ -305,8 +305,8 @@ def _build_covmat(s, bins, *, use_rho_tau): with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. """ pair = sacc_io._pair(bins) - idx_p = s.indices(sacc_io.XI_PLUS, pair) - idx_m = s.indices(sacc_io.XI_MINUS, pair) + idx_p = sacc_io._indices(s, sacc_io.XI_PLUS, pair, grid="reporting") + idx_m = sacc_io._indices(s, sacc_io.XI_MINUS, pair, grid="reporting") n_theta = len(idx_p) xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] @@ -322,8 +322,12 @@ def _build_covmat(s, bins, *, use_rho_tau): # SACC those two selections are not adjacent (τ_0− sits between them), so # gather both index sets and extract the joint sub-block, ξ↔τ zero. tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) - idx_tau0 = s.indices(sacc_io.TAU_PLUS.format(k=0), tau_pair) - idx_tau2 = s.indices(sacc_io.TAU_PLUS.format(k=2), tau_pair) + idx_tau0 = sacc_io._indices( + s, sacc_io.TAU_PLUS.format(k=0), tau_pair, grid="reporting" + ) + idx_tau2 = sacc_io._indices( + s, sacc_io.TAU_PLUS.format(k=2), tau_pair, grid="reporting" + ) tau_idx = np.concatenate([idx_tau0, idx_tau2]) tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] matrix = _block_diag(matrix, tau_cov) From 3e4fd7df4273c3348a0f55dcddbe211fcf69ac15 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 19 Jul 2026 14:43:56 +0200 Subject: [PATCH 35/38] refactor: merge twopoint_convert + one_covariance_io into sacc_interop One module whose job is converting between the SACC product and external analysis-tool file formats (CosmoSIS 2pt-FITS, OneCovariance). Public API unchanged; sacc_io untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- src/sp_validation/one_covariance_io.py | 256 ----------------- .../{twopoint_convert.py => sacc_interop.py} | 268 +++++++++++++++++- ...py => test_sacc_interop_one_covariance.py} | 4 +- ...ldata.py => test_sacc_interop_realdata.py} | 6 +- ...nvert.py => test_sacc_interop_twopoint.py} | 32 +-- 5 files changed, 277 insertions(+), 289 deletions(-) delete mode 100644 src/sp_validation/one_covariance_io.py rename src/sp_validation/{twopoint_convert.py => sacc_interop.py} (58%) rename src/sp_validation/tests/{test_one_covariance_io.py => test_sacc_interop_one_covariance.py} (99%) rename src/sp_validation/tests/{test_twopoint_convert_realdata.py => test_sacc_interop_realdata.py} (98%) rename src/sp_validation/tests/{test_twopoint_convert.py => test_sacc_interop_twopoint.py} (95%) diff --git a/src/sp_validation/one_covariance_io.py b/src/sp_validation/one_covariance_io.py deleted file mode 100644 index 4c0f2ae4..00000000 --- a/src/sp_validation/one_covariance_io.py +++ /dev/null @@ -1,256 +0,0 @@ -"""ONE_COVARIANCE_IO. - -:Name: one_covariance_io.py - -:Description: File-format glue between the SACC data-product layout - (:mod:`sp_validation.sacc_io`) and OneCovariance - (https://github.com/rreischke/OneCovariance). Two directions: - - - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the - ``source_i`` NZ tracers of an analysis SACC are written as the - combined whitespace-delimited redshift file OneCovariance reads - (column 0 = z grid, then one ``n(z)`` column per tomographic - bin, no bin edges), and a matching ``[redshift]`` config stanza - is returned via :func:`nz_config_stanza`. - - - **OneCovariance output -> SACC covariance blocks** - (:func:`covariance_blocks`): the flat ``covariance_list_*.dat`` - table OneCovariance emits (one row per element pair) is reshaped - into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the - per-block reshape — and paired with SACC selectors so a caller - can feed them straight to - :func:`sp_validation.sacc_io.assemble_covariance`. - - OneCovariance itself is *not* a dependency: this module only - touches its file formats, verified against the upstream - ``config.ini`` (``rreischke/OneCovariance`` @ main). - - n(z) file format (upstream ``config.ini`` comment, verbatim): - - ``redshift n_1(z) ... n_{N_source}(z)`` - - i.e. a plain whitespace-delimited text file, column 0 the shared - redshift grid and one column per tomographic bin — no ``z_low``/ - ``z_high`` edges (this is the OneCovariance convention, distinct - from the CosmoSIS NZDATA table which *does* carry edges). All - source bins must therefore share one z grid. - - ``[redshift]`` config keys (upstream canonical names): a single - combined file goes in ``zlens_directory`` + ``zlens_file``; - ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where - in each histogram bin the tabulated ``n(z)`` value sits — ``mid`` - for the bin-centred grids the SACC stores. NOTE: the UNIONS - OneCovariance template driven by - ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older - key names ``z_directory``/``zlens_file`` instead; pass - ``dir_key="z_directory"`` to match that template. -""" - -import os - -import numpy as np - -from . import sacc_io -from .statistics import cov_from_one_covariance - - -def nz_table(s, n_bins): - """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. - - Parameters - ---------- - s : sacc.Sacc - SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. - n_bins : int - Number of tomographic source bins to write. - - Returns - ------- - numpy.ndarray - Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift - grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the - OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). - - Raises - ------ - ValueError - If any source bin is missing, or if the bins do not share one z grid - (OneCovariance's combined file has a single redshift column, so the - grids must agree bin-for-bin). - """ - z0, nz0 = sacc_io.get_nz(s, 0) - z0 = np.asarray(z0, dtype=float) - columns = [z0] - for i in range(n_bins): - if sacc_io.source_name(i) not in s.tracers: - raise ValueError( - f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " - f"a {n_bins}-bin OneCovariance n(z) file" - ) - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z0): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the " - "OneCovariance combined n(z) file has one shared redshift column" - ) - columns.append(np.asarray(nz_i, dtype=float)) - return np.column_stack(columns) - - -def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): - """Write the OneCovariance combined n(z) input file from a SACC. - - OneCovariance reads the source redshift distribution as a plain - whitespace-delimited text file whose column 0 is the shared redshift grid - and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) - … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the - SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza - that points OneCovariance at it. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC with the ``source_i`` NZ tracers. - path : str or pathlib.Path - Output text-file path (overwritten). Its directory + basename become - the ``[redshift]`` directory/file config values. - n_bins : int - Number of tomographic source bins to write. - dir_key : str, optional - Config key for the redshift directory. Default ``"zlens_directory"`` - (upstream canonical). Pass ``"z_directory"`` for the UNIONS template - driven by ``pseudo_cl.py._modify_onecov_config``. - header : bool, optional - If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header - naming the columns; OneCovariance's ``genfromtxt``-style reader ignores - it. Set ``False`` for a bare numeric file. - - Returns - ------- - dict - The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming - the file just written. - """ - table = nz_table(s, n_bins) - head = "" - if header: - cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) - head = f"redshift {cols}" - np.savetxt(str(path), table, header=head) - return nz_config_stanza( - os.path.dirname(os.path.abspath(str(path))), - os.path.basename(str(path)), - dir_key=dir_key, - ) - - -def nz_config_stanza( - directory, filename, *, dir_key="zlens_directory", value_loc="mid" -): - """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. - - Parameters - ---------- - directory : str - Directory holding the n(z) file (OneCovariance ``*_directory`` value). - filename : str - n(z) file basename (OneCovariance ``zlens_file`` value). - dir_key : str, optional - Directory config key — ``"zlens_directory"`` (upstream) or - ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. - value_loc : str, optional - ``value_loc_in_lensbin`` — where in each histogram bin the tabulated - ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, - matching the bin-centred grids the SACC stores. - - Returns - ------- - dict - The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": - filename, "value_loc_in_lensbin": value_loc}``. Assign these under - ``config["redshift"]`` of a OneCovariance ``configparser`` config. - """ - if value_loc not in ("mid", "left", "right"): - raise ValueError( - f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" - ) - return { - dir_key: directory, - "zlens_file": filename, - "value_loc_in_lensbin": value_loc, - } - - -def read_nz(path): - """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. - - Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is - not stored in the file). Comment/header lines are skipped. - - Parameters - ---------- - path : str or pathlib.Path - n(z) text file (column 0 = z, columns 1… = per-bin n(z)). - - Returns - ------- - tuple - ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) - and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). - """ - table = np.atleast_2d(np.genfromtxt(str(path))) - return table[:, 0], table[:, 1:] - - -def covariance_blocks(cov_list, selectors, *, gaussian=True): - """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. - - OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per - ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value - lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This - reshapes the flat table into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block - reshape — and pairs each with its SACC selector, ready for - :func:`sp_validation.sacc_io.assemble_covariance`. - - Single-statistic case: pass the whole table and one selector; you get one - ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a - sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a - contiguous slice of the flat output for one statistic / bin-pair — and each - is reshaped and re-paired with its selector in order. The API is thus shaped - to extend to multi-probe blocking without over-fitting the single-bin case. - - Parameters - ---------- - cov_list : numpy.ndarray or sequence - Either the flat OneCovariance table (2-D array, one row per pair) for a - single block, or — for the multi-block form — a sequence of - ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` - must be ``None`` (the selectors travel with the sub-tables). - selectors : selector or None - For the single-block form, the SACC selector for the whole table (a - ``(data_type, tracers[, tags])`` tuple or an index array, as - :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the - multi-block form. - gaussian : bool, optional - Select the Gaussian-only column (``True``, default) or the - Gaussian+non-Gaussian column (``False``); passed straight through to - ``cov_from_one_covariance``. - - Returns - ------- - list - Ordered ``(selector, dense_cov)`` pairs, directly consumable by - ``sacc_io.assemble_covariance(s, blocks)``. - """ - if selectors is None: - # Multi-block form: cov_list is a sequence of (selector, sub_table). - return [ - (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) - for selector, sub in cov_list - ] - # Single-block form: one flat table, one selector. - return [ - (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) - ] diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/sacc_interop.py similarity index 58% rename from src/sp_validation/twopoint_convert.py rename to src/sp_validation/sacc_interop.py index 5e3c3156..977470a7 100644 --- a/src/sp_validation/twopoint_convert.py +++ b/src/sp_validation/sacc_interop.py @@ -1,16 +1,23 @@ -"""TWOPOINT_CONVERT. +"""SACC_INTEROP. -:Name: twopoint_convert.py +:Name: sacc_interop.py -:Description: Convert an analysis SACC file into the "2pt FITS" that CosmoSIS's - ``2pt_like`` (and Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) - reads. The output reproduces today's hand-assembled product from +:Description: Converters between the SACC data product + (:mod:`sp_validation.sacc_io`) and external analysis-tool file + formats — the CosmoSIS "2pt FITS" and OneCovariance's redshift / + covariance files. Two independent, self-contained sections; the + only shared surface is the SACC on one side. + + **CosmoSIS 2pt-FITS** (:func:`sacc_to_twopoint_fits`). Convert an + analysis SACC into the "2pt FITS" that CosmoSIS's ``2pt_like`` (and + Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) reads. The output + reproduces today's hand-assembled product from ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and - byte-for-byte: an NZDATA table, XI_PLUS / XI_MINUS 2pt tables, - optional CELL_EE / CELL_BB pseudo-Cℓ tables, the blocked COVMAT - (with ``STRT_i`` block-offset headers) and separate COVMAT_CELL, - and — when the SACC carries them — the TAU_{0,2}_PLUS 2pt tables - and the RHO_STATS table. + byte-for-byte (verified against that writer): an NZDATA table, + XI_PLUS / XI_MINUS 2pt tables, optional CELL_EE / CELL_BB pseudo-Cℓ + tables, the blocked COVMAT (with ``STRT_i`` block-offset headers) + and separate COVMAT_CELL, and — when the SACC carries them — the + TAU_{0,2}_PLUS 2pt tables and the RHO_STATS table. The converter is the *inverse* of the SACC writers in :mod:`sp_validation.sacc_io`: it reads statistics back through @@ -22,8 +29,9 @@ Scope note (single-bin today, tomography-ready): the assembly this mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ - ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly. - A tomographic 2pt-FITS layout (multiple bin pairs, per-pair + ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly and + fails fast on a multi-bin SACC (single-bin v1 contract). A + tomographic 2pt-FITS layout (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) is a later extension; it is not what today's CosmoSIS pipeline consumes, so it is out of scope for the byte-compatible converter. @@ -37,12 +45,42 @@ file today's assembly copies verbatim); it never fabricates variances. ξ±, Cℓ, n(z) and the covariance — the data vector CosmoSIS fits — are fully reconstructed from SACC alone. + + **OneCovariance** (:func:`write_nz`, :func:`nz_config_stanza`, + :func:`read_nz`, :func:`covariance_blocks`). File-format glue + between the SACC layout and OneCovariance + (https://github.com/rreischke/OneCovariance). Two directions: the + ``source_i`` NZ tracers are written as the combined + whitespace-delimited redshift file OneCovariance reads (column 0 = + z grid, then one ``n(z)`` column per bin, no bin edges) with a + matching ``[redshift]`` config stanza; and the flat + ``covariance_list_*.dat`` table OneCovariance emits is reshaped + into dense SACC covariance block(s) paired with their selectors. + OneCovariance itself is *not* a dependency — this module only + touches its file formats, verified against the upstream + ``config.ini`` (``rreischke/OneCovariance`` @ main). + + n(z) file format (upstream ``config.ini`` comment, verbatim): + + ``redshift n_1(z) ... n_{N_source}(z)`` + + i.e. a plain whitespace-delimited text file, column 0 the shared + redshift grid and one column per tomographic bin — no ``z_low``/ + ``z_high`` edges (distinct from the CosmoSIS NZDATA table above, + which *does* carry edges). All source bins must share one z grid. """ +import os + import numpy as np from astropy.io import fits from . import sacc_io +from .statistics import cov_from_one_covariance + +# ============================================================================= +# CosmoSIS 2pt-FITS +# ============================================================================= # The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the # extension name — copied from cosmosis_fitting.py so the headers match card @@ -370,3 +408,209 @@ def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): rho_hdu.data = rho_hdu.data.copy() rho_hdu.data["theta"] = theta return (tau0_hdu, tau2_hdu), rho_hdu + + +# ============================================================================= +# OneCovariance +# ============================================================================= + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = sacc_io.get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if sacc_io.source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.sacc_io.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``sacc_io.assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_one_covariance_io.py b/src/sp_validation/tests/test_sacc_interop_one_covariance.py similarity index 99% rename from src/sp_validation/tests/test_one_covariance_io.py rename to src/sp_validation/tests/test_sacc_interop_one_covariance.py index 9bc7da56..f0df56e9 100644 --- a/src/sp_validation/tests/test_one_covariance_io.py +++ b/src/sp_validation/tests/test_sacc_interop_one_covariance.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`sp_validation.one_covariance_io`. +"""Tests for :mod:`sp_validation.sacc_interop`. All synthetic, all fast: the OneCovariance fixtures are built in memory shaped exactly like its real file I/O — a flat ``covariance_list`` table with @@ -22,7 +22,7 @@ import numpy.testing as npt import pytest -from sp_validation import one_covariance_io as ocio +from sp_validation import sacc_interop as ocio from sp_validation import sacc_io as sio diff --git a/src/sp_validation/tests/test_twopoint_convert_realdata.py b/src/sp_validation/tests/test_sacc_interop_realdata.py similarity index 98% rename from src/sp_validation/tests/test_twopoint_convert_realdata.py rename to src/sp_validation/tests/test_sacc_interop_realdata.py index 1dcf920f..df7b1366 100644 --- a/src/sp_validation/tests/test_twopoint_convert_realdata.py +++ b/src/sp_validation/tests/test_sacc_interop_realdata.py @@ -28,7 +28,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_io, twopoint_convert +from sp_validation import sacc_interop, sacc_io _DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") _REAL_FILES = { @@ -221,7 +221,7 @@ def test_realdata_roundtrip_byte_equal(label, tmp_path): reference = _current_script_reference(cf, hdul, tmp_path) converted = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -250,7 +250,7 @@ def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) ondisk_names = [h.name for h in hdul] converted = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(converted) as conv: diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_sacc_interop_twopoint.py similarity index 95% rename from src/sp_validation/tests/test_twopoint_convert.py rename to src/sp_validation/tests/test_sacc_interop_twopoint.py index 59e0256d..cd1c4a31 100644 --- a/src/sp_validation/tests/test_twopoint_convert.py +++ b/src/sp_validation/tests/test_sacc_interop_twopoint.py @@ -1,6 +1,6 @@ """Byte-compare tests for the SACC -> 2pt-FITS converter. -The converter (:mod:`sp_validation.twopoint_convert`) must reproduce the CosmoSIS +The converter (:mod:`sp_validation.sacc_interop`) must reproduce the CosmoSIS 2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau ``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check @@ -36,7 +36,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_io, twopoint_convert +from sp_validation import sacc_interop, sacc_io _SCRIPT = ( Path(__file__).resolve().parents[3] @@ -290,7 +290,7 @@ def test_plain_xi_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp) s = _sacc(inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -300,7 +300,7 @@ def test_xi_cl_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp, cl=True) s = _sacc(inp, cl=True) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -311,7 +311,7 @@ def test_xi_rho_tau_byte_equal(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) assert out.read_bytes() == reference.read_bytes() @@ -334,7 +334,7 @@ def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(out) as hdul: @@ -351,7 +351,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp = _inputs(seed=0) s = _sacc(inp) out = tmp_path / "base.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) with fits.open(out) as hdul: base_xip = hdul["XI_PLUS"].data["VALUE"].copy() @@ -359,7 +359,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp2["xip"] = inp2["xip"] + 1.0 s2 = _sacc(inp2) out2 = tmp_path / "perturbed.fits" - twopoint_convert.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) with fits.open(out2) as hdul: new_xip = hdul["XI_PLUS"].data["VALUE"] @@ -384,7 +384,7 @@ def test_integration_grid_points_ignored(tmp_path): s_plain = _sacc(inp, cl=True, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out_plain = tmp_path / "plain.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -452,7 +452,7 @@ def test_integration_grid_points_ignored(tmp_path): s_aug.add_covariance(full) out_aug = tmp_path / "aug.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -465,7 +465,7 @@ def test_rho_tau_sidecars_required_together(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) with pytest.raises(ValueError, match="together"): - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 ) @@ -490,9 +490,9 @@ def test_tomographic_sacc_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="single-bin only"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) with pytest.raises(ValueError, match="single-bin only"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) assert not (tmp_path / "x.fits").exists() @@ -513,7 +513,7 @@ def test_sacc_without_xi_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="nothing to convert"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) assert not (tmp_path / "x.fits").exists() @@ -538,7 +538,7 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "encoded.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -567,7 +567,7 @@ def _mask_idx(dtype, tracers): _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) - expected = twopoint_convert._block_diag( + expected = sacc_interop._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) cell_idx = _mask_idx(sacc_io.CL_EE, pair) From 33f0321ea81fe33415488ec33ee6e08b2cefcacf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 20 Jul 2026 22:07:55 +0200 Subject: [PATCH 36/38] =?UTF-8?q?refactor:=20fold=20sacc=5Finterop=20into?= =?UTF-8?q?=20sacc=5Fio=20=E2=80=94=20one=20SACC=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cail's ruling: a single sacc_io.py carries the format contract and the external-tool converters (CosmoSIS 2pt-FITS, OneCovariance). Public API unchanged; the sacc_io import name every downstream branch uses is untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- src/sp_validation/sacc_interop.py | 616 ------------------ src/sp_validation/sacc_io.py | 552 ++++++++++++++++ ...ance.py => test_sacc_io_one_covariance.py} | 29 +- ...p_realdata.py => test_sacc_io_realdata.py} | 6 +- ...p_twopoint.py => test_sacc_io_twopoint.py} | 32 +- 5 files changed, 585 insertions(+), 650 deletions(-) delete mode 100644 src/sp_validation/sacc_interop.py rename src/sp_validation/tests/{test_sacc_interop_one_covariance.py => test_sacc_io_one_covariance.py} (92%) rename src/sp_validation/tests/{test_sacc_interop_realdata.py => test_sacc_io_realdata.py} (98%) rename src/sp_validation/tests/{test_sacc_interop_twopoint.py => test_sacc_io_twopoint.py} (96%) diff --git a/src/sp_validation/sacc_interop.py b/src/sp_validation/sacc_interop.py deleted file mode 100644 index 977470a7..00000000 --- a/src/sp_validation/sacc_interop.py +++ /dev/null @@ -1,616 +0,0 @@ -"""SACC_INTEROP. - -:Name: sacc_interop.py - -:Description: Converters between the SACC data product - (:mod:`sp_validation.sacc_io`) and external analysis-tool file - formats — the CosmoSIS "2pt FITS" and OneCovariance's redshift / - covariance files. Two independent, self-contained sections; the - only shared surface is the SACC on one side. - - **CosmoSIS 2pt-FITS** (:func:`sacc_to_twopoint_fits`). Convert an - analysis SACC into the "2pt FITS" that CosmoSIS's ``2pt_like`` (and - Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) reads. The output - reproduces today's hand-assembled product from - ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and - byte-for-byte (verified against that writer): an NZDATA table, - XI_PLUS / XI_MINUS 2pt tables, optional CELL_EE / CELL_BB pseudo-Cℓ - tables, the blocked COVMAT (with ``STRT_i`` block-offset headers) - and separate COVMAT_CELL, and — when the SACC carries them — the - TAU_{0,2}_PLUS 2pt tables and the RHO_STATS table. - - The converter is the *inverse* of the SACC writers in - :mod:`sp_validation.sacc_io`: it reads statistics back through - those readers and lays them into the DES ``twopoint`` FITS - convention. SACC's canonical order is pair-major (per pair - ``[ξ+; ξ−]``); the 2pt-FITS layout is type-major (all ξ+, then all - ξ−), so the data-vector and its covariance are permuted here via - ``s.indices`` rather than assuming any global order. - - Scope note (single-bin today, tomography-ready): the assembly this - mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ - ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly and - fails fast on a multi-bin SACC (single-bin v1 contract). A - tomographic 2pt-FITS layout (multiple bin pairs, per-pair - BIN1/BIN2, one NZ column per bin) is a later extension; it is not - what today's CosmoSIS pipeline consumes, so it is out of scope for - the byte-compatible converter. - - Rho/tau caveat: the SACC layout stores ρ±/τ± *values* only, while - the 2pt-FITS RHO_STATS table also carries the per-mode *variances* - (``varrho_*``) that Sacha's fork's covariance path reads. Those - variances are not recoverable from the analysis SACC. The - converter therefore writes RHO_STATS / TAU HDUs only when a - ``rho_stats``/``tau_stats`` sidecar FITS is supplied (the same - file today's assembly copies verbatim); it never fabricates - variances. ξ±, Cℓ, n(z) and the covariance — the data vector - CosmoSIS fits — are fully reconstructed from SACC alone. - - **OneCovariance** (:func:`write_nz`, :func:`nz_config_stanza`, - :func:`read_nz`, :func:`covariance_blocks`). File-format glue - between the SACC layout and OneCovariance - (https://github.com/rreischke/OneCovariance). Two directions: the - ``source_i`` NZ tracers are written as the combined - whitespace-delimited redshift file OneCovariance reads (column 0 = - z grid, then one ``n(z)`` column per bin, no bin edges) with a - matching ``[redshift]`` config stanza; and the flat - ``covariance_list_*.dat`` table OneCovariance emits is reshaped - into dense SACC covariance block(s) paired with their selectors. - OneCovariance itself is *not* a dependency — this module only - touches its file formats, verified against the upstream - ``config.ini`` (``rreischke/OneCovariance`` @ main). - - n(z) file format (upstream ``config.ini`` comment, verbatim): - - ``redshift n_1(z) ... n_{N_source}(z)`` - - i.e. a plain whitespace-delimited text file, column 0 the shared - redshift grid and one column per tomographic bin — no ``z_low``/ - ``z_high`` edges (distinct from the CosmoSIS NZDATA table above, - which *does* carry edges). All source bins must share one z grid. -""" - -import os - -import numpy as np -from astropy.io import fits - -from . import sacc_io -from .statistics import cov_from_one_covariance - -# ============================================================================= -# CosmoSIS 2pt-FITS -# ============================================================================= - -# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the -# extension name — copied from cosmosis_fitting.py so the headers match card -# for card. -_QUANT = { - "XI_PLUS": ("G+R", "G+R"), - "XI_MINUS": ("G-R", "G-R"), - "CELL_EE": ("GEF", "GEF"), - "CELL_BB": ("GBF", "GBF"), - "TAU_0_PLUS": ("G+R", "P+R"), - "TAU_2_PLUS": ("G+R", "SR+R"), -} - - -def _twopoint_hdu(name, values, ang, *, ang_unit=None): - """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). - - Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: - same column order and formats, the ``2PTDATA`` marker, the QUANT pair for - ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG - column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). - """ - nbins = len(values) - angbin = np.arange(1, nbins + 1) - columns = [ - fits.Column(name="BIN1", format="K", array=np.ones(nbins)), - fits.Column(name="BIN2", format="K", array=np.ones(nbins)), - fits.Column(name="ANGBIN", format="K", array=angbin), - fits.Column(name="VALUE", format="D", array=values), - fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), - ] - hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) - quant1, quant2 = _QUANT[name] - for key, value in { - "2PTDATA": "T", - "QUANT1": quant1, - "QUANT2": quant2, - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - }.items(): - hdu.header[key] = value - return hdu - - -def _nz_hdu(s, n_bins): - """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. - - Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` - grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` - column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins - are required to share the ``z`` grid — the single ``Z_MID`` axis of the - DES NZDATA table. - """ - z_mid, nz0 = sacc_io.get_nz(s, 0) - z_mid = np.asarray(z_mid, dtype=float) - step = z_mid[1] - z_mid[0] - z_low = z_mid - step / 2 - z_high = z_mid + step / 2 - - columns = [ - fits.Column(name="Z_LOW", format="D", array=z_low), - fits.Column(name="Z_MID", format="D", array=z_mid), - fits.Column(name="Z_HIGH", format="D", array=z_high), - ] - for i in range(n_bins): - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the DES " - "NZDATA table requires one shared Z_MID axis" - ) - columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) - - hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") - for key, value in { - "NZDATA": "T ", - "EXTNAME": "NZ_SOURCE", - "NBIN": n_bins, - "NZ": len(z_low), - }.items(): - hdu.header[key] = value - return hdu - - -def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): - """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. - - Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for - card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed - and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` - CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME - card is created early, with astropy's standard comment) before re-stamping. - ``name_in_ctor`` selects the second form so the card order matches exactly. - """ - matrix = np.asarray(matrix, dtype=np.float64) - if matrix.shape[0] != matrix.shape[1]: - raise ValueError(f"covariance must be square; got shape {matrix.shape}") - hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) - hdu.header["COVDATA"] = "True" - hdu.header["EXTNAME"] = extname - for i, (name, start) in enumerate(zip(block_names, block_starts)): - hdu.header[f"NAME_{i}"] = name - hdu.header[f"STRT_{i}"] = int(start) - return hdu - - -def _type_major_xi(s, bins): - """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. - - ``sacc_io.get_xi`` already returns each statistic in insertion (= ascending - θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two - arrays it hands back, so no further permutation is needed for a single pair. - """ - return sacc_io.get_xi(s, bins, grid="reporting") - - -def _require_single_bin(s, n_bins): - """Fail fast unless the SACC is a valid single-bin ξ product. - - The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads - (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip - through silently — ``n_bins`` alone drives the NZDATA column count while the - ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a - ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. - Guards both the empty-ξ case and the single-bin contract; tomographic - emission lands with the tomographic round. - """ - pairs = s.get_tracer_combinations(sacc_io.XI_PLUS) - if not pairs: - raise ValueError( - f"SACC has no {sacc_io.XI_PLUS} points — nothing to convert; the " - "2pt-FITS data vector is built from the ξ± statistics" - ) - expected = (sacc_io.source_name(0), sacc_io.source_name(0)) - if n_bins != 1 or set(pairs) != {expected}: - raise ValueError( - f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " - f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " - "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " - "bin) lands with the tomographic round." - ) - - -def sacc_to_twopoint_fits( - s, - path, - *, - rho_stats_hdu=None, - tau_stats_hdu=None, - n_bins=1, -): - """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. - - The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product - for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then - (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, - and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, - RHO_STATS. The data vector and its covariance are laid out type-major - (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` - convention CosmoSIS reads. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the - ρ/τ product — the τ data points; see ``rho_stats_hdu``). - path : str - Output FITS path (overwritten). - rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional - The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's - assembly does. Required together to write the ρ/τ product; the SACC - alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When - omitted, a pure ξ (± Cℓ) product is written. - n_bins : int, optional - Number of source tomographic bins. Must be ``1``: this converter emits - the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic - emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) - lands with the tomographic round; the converter fails fast on anything - else rather than silently truncating to bin ``(0, 0)``. - - Returns - ------- - astropy.io.fits.HDUList - The assembled list, also written to ``path``. - - Raises - ------ - ValueError - If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer - pairs are anything other than exactly ``{(source_0, source_0)}`` (the - single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. - """ - if (rho_stats_hdu is None) != (tau_stats_hdu is None): - raise ValueError( - "rho_stats_hdu and tau_stats_hdu must be supplied together " - "(the ρ/τ product needs both, or neither for a pure-ξ product)" - ) - _require_single_bin(s, n_bins) - use_rho_tau = rho_stats_hdu is not None - bins = (0, 0) - - nz_hdu = _nz_hdu(s, n_bins) - theta, xip, xim = _type_major_xi(s, bins) - xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") - xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") - - cell_hdu, cov_cell_hdu = _build_cell(s, bins) - - cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) - - tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) - - # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, - # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. - hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] - if cov_cell_hdu is not None: - hdu_list.append(cov_cell_hdu) - hdu_list.extend([xip_hdu, xim_hdu]) - if cell_hdu is not None: - hdu_list.append(cell_hdu) - if use_rho_tau: - hdu_list.extend([*tau_hdus, rho_hdu]) - - hdul = fits.HDUList(hdu_list) - hdul.writeto(path, overwrite=True) - return hdul - - -def _build_cell(s, bins): - """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. - - Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is - emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's - assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). - The SACC still carries EE/BB/EB with bandpower windows for the B-mode - null-test path; this converter surfaces only the block CosmoSIS reads. The - CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL - ImageHDU, matching today's product. - """ - if sacc_io.CL_EE not in s.get_data_types(): - return None, None - - ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) - cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) - cell_idx = sacc_io._indices(s, sacc_io.CL_EE, sacc_io._pair(bins), grid="reporting") - cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] - cov_cell_hdu = _cov_hdu( - cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True - ) - return cell_hdu, cov_cell_hdu - - -def _build_covmat(s, bins, *, use_rho_tau): - """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). - - The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for - the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under - ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally - with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. - """ - pair = sacc_io._pair(bins) - idx_p = sacc_io._indices(s, sacc_io.XI_PLUS, pair, grid="reporting") - idx_m = sacc_io._indices(s, sacc_io.XI_MINUS, pair, grid="reporting") - n_theta = len(idx_p) - xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation - xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] - - names = ["XI_PLUS", "XI_MINUS"] - starts = [0, n_theta] - matrix = xi_cov - - if use_rho_tau: - # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the - # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in - # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the - # SACC those two selections are not adjacent (τ_0− sits between them), so - # gather both index sets and extract the joint sub-block, ξ↔τ zero. - tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) - idx_tau0 = sacc_io._indices( - s, sacc_io.TAU_PLUS.format(k=0), tau_pair, grid="reporting" - ) - idx_tau2 = sacc_io._indices( - s, sacc_io.TAU_PLUS.format(k=2), tau_pair, grid="reporting" - ) - tau_idx = np.concatenate([idx_tau0, idx_tau2]) - tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] - matrix = _block_diag(matrix, tau_cov) - names += ["TAU_0_PLUS", "TAU_2_PLUS"] - starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] - - return _cov_hdu(matrix, names, starts) - - -def _block_diag(*blocks): - """Stack square blocks block-diagonally with zero cross-blocks.""" - sizes = [b.shape[0] for b in blocks] - n = sum(sizes) - out = np.zeros((n, n)) - start = 0 - for b in blocks: - out[start : start + b.shape[0], start : start + b.shape[0]] = b - start += b.shape[0] - return out - - -def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): - """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. - - Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` - columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied - verbatim from the sidecar with its θ column forced onto the ξ grid. The - ``varrho_*`` columns ride along in the copy — they are why the sidecar is - required (the SACC cannot supply them). - """ - if not use_rho_tau: - return (), None - - tau = tau_stats_hdu.data - tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") - tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") - - rho_hdu = rho_stats_hdu.copy() - rho_hdu.name = "RHO_STATS" - rho_hdu.data = rho_hdu.data.copy() - rho_hdu.data["theta"] = theta - return (tau0_hdu, tau2_hdu), rho_hdu - - -# ============================================================================= -# OneCovariance -# ============================================================================= - - -def nz_table(s, n_bins): - """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. - - Parameters - ---------- - s : sacc.Sacc - SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. - n_bins : int - Number of tomographic source bins to write. - - Returns - ------- - numpy.ndarray - Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift - grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the - OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). - - Raises - ------ - ValueError - If any source bin is missing, or if the bins do not share one z grid - (OneCovariance's combined file has a single redshift column, so the - grids must agree bin-for-bin). - """ - z0, nz0 = sacc_io.get_nz(s, 0) - z0 = np.asarray(z0, dtype=float) - columns = [z0] - for i in range(n_bins): - if sacc_io.source_name(i) not in s.tracers: - raise ValueError( - f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " - f"a {n_bins}-bin OneCovariance n(z) file" - ) - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z0): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the " - "OneCovariance combined n(z) file has one shared redshift column" - ) - columns.append(np.asarray(nz_i, dtype=float)) - return np.column_stack(columns) - - -def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): - """Write the OneCovariance combined n(z) input file from a SACC. - - OneCovariance reads the source redshift distribution as a plain - whitespace-delimited text file whose column 0 is the shared redshift grid - and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) - … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the - SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza - that points OneCovariance at it. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC with the ``source_i`` NZ tracers. - path : str or pathlib.Path - Output text-file path (overwritten). Its directory + basename become - the ``[redshift]`` directory/file config values. - n_bins : int - Number of tomographic source bins to write. - dir_key : str, optional - Config key for the redshift directory. Default ``"zlens_directory"`` - (upstream canonical). Pass ``"z_directory"`` for the UNIONS template - driven by ``pseudo_cl.py._modify_onecov_config``. - header : bool, optional - If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header - naming the columns; OneCovariance's ``genfromtxt``-style reader ignores - it. Set ``False`` for a bare numeric file. - - Returns - ------- - dict - The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming - the file just written. - """ - table = nz_table(s, n_bins) - head = "" - if header: - cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) - head = f"redshift {cols}" - np.savetxt(str(path), table, header=head) - return nz_config_stanza( - os.path.dirname(os.path.abspath(str(path))), - os.path.basename(str(path)), - dir_key=dir_key, - ) - - -def nz_config_stanza( - directory, filename, *, dir_key="zlens_directory", value_loc="mid" -): - """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. - - Parameters - ---------- - directory : str - Directory holding the n(z) file (OneCovariance ``*_directory`` value). - filename : str - n(z) file basename (OneCovariance ``zlens_file`` value). - dir_key : str, optional - Directory config key — ``"zlens_directory"`` (upstream) or - ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. - value_loc : str, optional - ``value_loc_in_lensbin`` — where in each histogram bin the tabulated - ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, - matching the bin-centred grids the SACC stores. - - Returns - ------- - dict - The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": - filename, "value_loc_in_lensbin": value_loc}``. Assign these under - ``config["redshift"]`` of a OneCovariance ``configparser`` config. - """ - if value_loc not in ("mid", "left", "right"): - raise ValueError( - f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" - ) - return { - dir_key: directory, - "zlens_file": filename, - "value_loc_in_lensbin": value_loc, - } - - -def read_nz(path): - """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. - - Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is - not stored in the file). Comment/header lines are skipped. - - Parameters - ---------- - path : str or pathlib.Path - n(z) text file (column 0 = z, columns 1… = per-bin n(z)). - - Returns - ------- - tuple - ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) - and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). - """ - table = np.atleast_2d(np.genfromtxt(str(path))) - return table[:, 0], table[:, 1:] - - -def covariance_blocks(cov_list, selectors, *, gaussian=True): - """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. - - OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per - ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value - lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This - reshapes the flat table into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block - reshape — and pairs each with its SACC selector, ready for - :func:`sp_validation.sacc_io.assemble_covariance`. - - Single-statistic case: pass the whole table and one selector; you get one - ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a - sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a - contiguous slice of the flat output for one statistic / bin-pair — and each - is reshaped and re-paired with its selector in order. The API is thus shaped - to extend to multi-probe blocking without over-fitting the single-bin case. - - Parameters - ---------- - cov_list : numpy.ndarray or sequence - Either the flat OneCovariance table (2-D array, one row per pair) for a - single block, or — for the multi-block form — a sequence of - ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` - must be ``None`` (the selectors travel with the sub-tables). - selectors : selector or None - For the single-block form, the SACC selector for the whole table (a - ``(data_type, tracers[, tags])`` tuple or an index array, as - :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the - multi-block form. - gaussian : bool, optional - Select the Gaussian-only column (``True``, default) or the - Gaussian+non-Gaussian column (``False``); passed straight through to - ``cov_from_one_covariance``. - - Returns - ------- - list - Ordered ``(selector, dense_cov)`` pairs, directly consumable by - ``sacc_io.assemble_covariance(s, blocks)``. - """ - if selectors is None: - # Multi-block form: cov_list is a sequence of (selector, sub_table). - return [ - (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) - for selector, sub in cov_list - ] - # Single-block form: one flat table, one selector. - return [ - (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) - ] diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 3df6488a..355f86c1 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -49,10 +49,28 @@ the covariance was built in. Converters that need a type-major layout (e.g. the DES 2pt-FITS convention) permute explicitly via ``s.indices`` rather than assuming global order. + + **Converters.** The tail of this module holds converters between + the SACC layout above and external analysis-tool file formats — + the CosmoSIS "2pt FITS" (``sacc_to_twopoint_fits``) and + OneCovariance's redshift / covariance files (``write_nz``, + ``nz_config_stanza``, ``read_nz``, ``covariance_blocks``). The + 2pt-FITS converter reproduces today's hand-assembled product from + ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and + byte-for-byte (verified against that writer), and is single-bin + only today — it fails fast on a multi-bin SACC (tomographic + emission lands with the tomographic round). The OneCovariance + converters are coupled to SACC by file format only; OneCovariance + itself is not a dependency. """ +import os + import numpy as np import sacc +from astropy.io import fits + +from .statistics import cov_from_one_covariance PSF_TRACER = "psf_stars" @@ -887,3 +905,537 @@ def load(path, *, allow_unblinded=False): "allow_unblinded=True." ) return s + + +# ============================================================================= +# CosmoSIS 2pt-FITS +# ============================================================================= + +# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the +# extension name — copied from cosmosis_fitting.py so the headers match card +# for card. +_QUANT = { + "XI_PLUS": ("G+R", "G+R"), + "XI_MINUS": ("G-R", "G-R"), + "CELL_EE": ("GEF", "GEF"), + "CELL_BB": ("GBF", "GBF"), + "TAU_0_PLUS": ("G+R", "P+R"), + "TAU_2_PLUS": ("G+R", "SR+R"), +} + + +def _twopoint_hdu(name, values, ang, *, ang_unit=None): + """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). + + Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: + same column order and formats, the ``2PTDATA`` marker, the QUANT pair for + ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG + column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). + """ + nbins = len(values) + angbin = np.arange(1, nbins + 1) + columns = [ + fits.Column(name="BIN1", format="K", array=np.ones(nbins)), + fits.Column(name="BIN2", format="K", array=np.ones(nbins)), + fits.Column(name="ANGBIN", format="K", array=angbin), + fits.Column(name="VALUE", format="D", array=values), + fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), + ] + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) + quant1, quant2 = _QUANT[name] + for key, value in { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + }.items(): + hdu.header[key] = value + return hdu + + +def _nz_hdu(s, n_bins): + """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. + + Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` + grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` + column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins + are required to share the ``z`` grid — the single ``Z_MID`` axis of the + DES NZDATA table. + """ + z_mid, nz0 = get_nz(s, 0) + z_mid = np.asarray(z_mid, dtype=float) + step = z_mid[1] - z_mid[0] + z_low = z_mid - step / 2 + z_high = z_mid + step / 2 + + columns = [ + fits.Column(name="Z_LOW", format="D", array=z_low), + fits.Column(name="Z_MID", format="D", array=z_mid), + fits.Column(name="Z_HIGH", format="D", array=z_high), + ] + for i in range(n_bins): + z_i, nz_i = get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the DES " + "NZDATA table requires one shared Z_MID axis" + ) + columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) + + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") + for key, value in { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": n_bins, + "NZ": len(z_low), + }.items(): + hdu.header[key] = value + return hdu + + +def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): + """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. + + Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for + card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed + and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` + CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME + card is created early, with astropy's standard comment) before re-stamping. + ``name_in_ctor`` selects the second form so the card order matches exactly. + """ + matrix = np.asarray(matrix, dtype=np.float64) + if matrix.shape[0] != matrix.shape[1]: + raise ValueError(f"covariance must be square; got shape {matrix.shape}") + hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) + hdu.header["COVDATA"] = "True" + hdu.header["EXTNAME"] = extname + for i, (name, start) in enumerate(zip(block_names, block_starts)): + hdu.header[f"NAME_{i}"] = name + hdu.header[f"STRT_{i}"] = int(start) + return hdu + + +def _type_major_xi(s, bins): + """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. + + ``get_xi`` already returns each statistic in insertion (= ascending + θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two + arrays it hands back, so no further permutation is needed for a single pair. + """ + return get_xi(s, bins, grid="reporting") + + +def _require_single_bin(s, n_bins): + """Fail fast unless the SACC is a valid single-bin ξ product. + + The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads + (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip + through silently — ``n_bins`` alone drives the NZDATA column count while the + ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a + ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. + Guards both the empty-ξ case and the single-bin contract; tomographic + emission lands with the tomographic round. + """ + pairs = s.get_tracer_combinations(XI_PLUS) + if not pairs: + raise ValueError( + f"SACC has no {XI_PLUS} points — nothing to convert; the " + "2pt-FITS data vector is built from the ξ± statistics" + ) + expected = (source_name(0), source_name(0)) + if n_bins != 1 or set(pairs) != {expected}: + raise ValueError( + f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " + f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " + "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " + "bin) lands with the tomographic round." + ) + + +def sacc_to_twopoint_fits( + s, + path, + *, + rho_stats_hdu=None, + tau_stats_hdu=None, + n_bins=1, +): + """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. + + The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product + for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then + (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, + and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, + RHO_STATS. The data vector and its covariance are laid out type-major + (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` + convention CosmoSIS reads. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the + ρ/τ product — the τ data points; see ``rho_stats_hdu``). + path : str + Output FITS path (overwritten). + rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional + The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's + assembly does. Required together to write the ρ/τ product; the SACC + alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When + omitted, a pure ξ (± Cℓ) product is written. + n_bins : int, optional + Number of source tomographic bins. Must be ``1``: this converter emits + the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic + emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) + lands with the tomographic round; the converter fails fast on anything + else rather than silently truncating to bin ``(0, 0)``. + + Returns + ------- + astropy.io.fits.HDUList + The assembled list, also written to ``path``. + + Raises + ------ + ValueError + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + pairs are anything other than exactly ``{(source_0, source_0)}`` (the + single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. + """ + if (rho_stats_hdu is None) != (tau_stats_hdu is None): + raise ValueError( + "rho_stats_hdu and tau_stats_hdu must be supplied together " + "(the ρ/τ product needs both, or neither for a pure-ξ product)" + ) + _require_single_bin(s, n_bins) + use_rho_tau = rho_stats_hdu is not None + bins = (0, 0) + + nz_hdu = _nz_hdu(s, n_bins) + theta, xip, xim = _type_major_xi(s, bins) + xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") + xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") + + cell_hdu, cov_cell_hdu = _build_cell(s, bins) + + cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) + + tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) + + # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, + # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. + hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] + if cov_cell_hdu is not None: + hdu_list.append(cov_cell_hdu) + hdu_list.extend([xip_hdu, xim_hdu]) + if cell_hdu is not None: + hdu_list.append(cell_hdu) + if use_rho_tau: + hdu_list.extend([*tau_hdus, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(path, overwrite=True) + return hdul + + +def _build_cell(s, bins): + """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. + + Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is + emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's + assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). + The SACC still carries EE/BB/EB with bandpower windows for the B-mode + null-test path; this converter surfaces only the block CosmoSIS reads. The + CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL + ImageHDU, matching today's product. + """ + if CL_EE not in s.get_data_types(): + return None, None + + ell, cl_ee, _cl_bb, _cl_eb, _window = get_pseudo_cl(s, bins) + cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) + cell_idx = _indices(s, CL_EE, _pair(bins), grid="reporting") + cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] + cov_cell_hdu = _cov_hdu( + cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True + ) + return cell_hdu, cov_cell_hdu + + +def _build_covmat(s, bins, *, use_rho_tau): + """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). + + The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for + the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under + ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally + with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. + """ + pair = _pair(bins) + idx_p = _indices(s, XI_PLUS, pair, grid="reporting") + idx_m = _indices(s, XI_MINUS, pair, grid="reporting") + n_theta = len(idx_p) + xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation + xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + names = ["XI_PLUS", "XI_MINUS"] + starts = [0, n_theta] + matrix = xi_cov + + if use_rho_tau: + # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the + # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in + # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the + # SACC those two selections are not adjacent (τ_0− sits between them), so + # gather both index sets and extract the joint sub-block, ξ↔τ zero. + tau_pair = (source_name(0), PSF_TRACER) + idx_tau0 = _indices(s, TAU_PLUS.format(k=0), tau_pair, grid="reporting") + idx_tau2 = _indices(s, TAU_PLUS.format(k=2), tau_pair, grid="reporting") + tau_idx = np.concatenate([idx_tau0, idx_tau2]) + tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] + matrix = _block_diag(matrix, tau_cov) + names += ["TAU_0_PLUS", "TAU_2_PLUS"] + starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] + + return _cov_hdu(matrix, names, starts) + + +def _block_diag(*blocks): + """Stack square blocks block-diagonally with zero cross-blocks.""" + sizes = [b.shape[0] for b in blocks] + n = sum(sizes) + out = np.zeros((n, n)) + start = 0 + for b in blocks: + out[start : start + b.shape[0], start : start + b.shape[0]] = b + start += b.shape[0] + return out + + +def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): + """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. + + Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` + columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied + verbatim from the sidecar with its θ column forced onto the ξ grid. The + ``varrho_*`` columns ride along in the copy — they are why the sidecar is + required (the SACC cannot supply them). + """ + if not use_rho_tau: + return (), None + + tau = tau_stats_hdu.data + tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") + tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") + + rho_hdu = rho_stats_hdu.copy() + rho_hdu.name = "RHO_STATS" + rho_hdu.data = rho_hdu.data.copy() + rho_hdu.data["theta"] = theta + return (tau0_hdu, tau2_hdu), rho_hdu + + +# ============================================================================= +# OneCovariance +# ============================================================================= + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_sacc_interop_one_covariance.py b/src/sp_validation/tests/test_sacc_io_one_covariance.py similarity index 92% rename from src/sp_validation/tests/test_sacc_interop_one_covariance.py rename to src/sp_validation/tests/test_sacc_io_one_covariance.py index f0df56e9..bbe90ca3 100644 --- a/src/sp_validation/tests/test_sacc_interop_one_covariance.py +++ b/src/sp_validation/tests/test_sacc_io_one_covariance.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`sp_validation.sacc_interop`. +"""Tests for :mod:`sp_validation.sacc_io`. All synthetic, all fast: the OneCovariance fixtures are built in memory shaped exactly like its real file I/O — a flat ``covariance_list`` table with @@ -22,7 +22,6 @@ import numpy.testing as npt import pytest -from sp_validation import sacc_interop as ocio from sp_validation import sacc_io as sio @@ -84,8 +83,8 @@ def test_covariance_blocks_reshapes_to_hand_built_matrix(): selector = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) - [(sel_g, block_g)] = ocio.covariance_blocks(table, selector, gaussian=True) - [(sel_a, block_a)] = ocio.covariance_blocks(table, selector, gaussian=False) + [(sel_g, block_g)] = sio.covariance_blocks(table, selector, gaussian=True) + [(sel_a, block_a)] = sio.covariance_blocks(table, selector, gaussian=False) assert sel_g == selector and sel_a == selector npt.assert_allclose(block_g, cov_gauss, rtol=1e-12) @@ -98,7 +97,7 @@ def test_covariance_blocks_reshapes_to_hand_built_matrix(): # entry (row k = i·n + j, col 10 for gaussian). perturbed = table.copy() perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) - [(_, block_p)] = ocio.covariance_blocks(perturbed, selector, gaussian=True) + [(_, block_p)] = sio.covariance_blocks(perturbed, selector, gaussian=True) npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) block_p[2, 1] = block_g[2, 1] npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved @@ -122,7 +121,7 @@ def test_covariance_blocks_multiblock_form(): sel_a = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) sel_b = (sio.XI_MINUS, (sio.source_name(0), sio.source_name(0))) - blocks = ocio.covariance_blocks( + blocks = sio.covariance_blocks( [(sel_a, table_a), (sel_b, table_b)], None, gaussian=True ) @@ -162,7 +161,7 @@ def test_covariance_blocks_feed_assemble_covariance(): [s.indices(sio.XI_PLUS, pair), s.indices(sio.XI_MINUS, pair)] ) - blocks = ocio.covariance_blocks(table, selector, gaussian=True) + blocks = sio.covariance_blocks(table, selector, gaussian=True) sio.assemble_covariance(s, blocks) npt.assert_allclose(s.covariance.dense, cov_gauss, rtol=1e-12) @@ -190,9 +189,9 @@ def test_write_nz_roundtrips_and_names_file(tmp_path): s = sio.new_sacc({0: (z, nz0), 1: (z, nz1)}) path = tmp_path / "nz_onecov.txt" - stanza = ocio.write_nz(s, path, n_bins=2) + stanza = sio.write_nz(s, path, n_bins=2) - z_read, nz_read = ocio.read_nz(path) + z_read, nz_read = sio.read_nz(path) npt.assert_allclose(z_read, z, rtol=1e-12) assert nz_read.shape == (len(z), 2) npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) @@ -213,7 +212,7 @@ def test_write_nz_unions_template_dir_key(tmp_path): """ z, nz0 = _nz(12) s = sio.new_sacc({0: (z, nz0)}) - stanza = ocio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") + stanza = sio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") assert "z_directory" in stanza and "zlens_directory" not in stanza assert stanza["z_directory"] == str(tmp_path) assert stanza["zlens_file"] == "nz.txt" @@ -231,7 +230,7 @@ def test_write_nz_fails_on_mismatched_z_grids(tmp_path): z1_shifted = z1_shifted + 0.1 # different grid s = sio.new_sacc({0: (z0, nz0), 1: (z1_shifted, nz1)}) with pytest.raises(ValueError, match="differs from source bin 0"): - ocio.write_nz(s, tmp_path / "bad.txt", n_bins=2) + sio.write_nz(s, tmp_path / "bad.txt", n_bins=2) def test_write_nz_no_header_roundtrips(tmp_path): @@ -244,8 +243,8 @@ def test_write_nz_no_header_roundtrips(tmp_path): z, nz0 = _nz(40) s = sio.new_sacc({0: (z, nz0)}) path = tmp_path / "bare.txt" - ocio.write_nz(s, path, n_bins=1, header=False) - z_read, nz_read = ocio.read_nz(path) + sio.write_nz(s, path, n_bins=1, header=False) + z_read, nz_read = sio.read_nz(path) npt.assert_allclose(z_read, z, rtol=1e-12) npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) @@ -258,7 +257,7 @@ def test_nz_config_stanza_rejects_bad_value_loc(): raise ``ValueError`` rather than write a stanza OneCovariance will reject. """ with pytest.raises(ValueError, match="value_loc_in_lensbin"): - ocio.nz_config_stanza("/dir", "nz.txt", value_loc="center") + sio.nz_config_stanza("/dir", "nz.txt", value_loc="center") def test_write_nz_fails_on_missing_bin(tmp_path): @@ -271,4 +270,4 @@ def test_write_nz_fails_on_missing_bin(tmp_path): z, nz0 = _nz(30) s = sio.new_sacc({0: (z, nz0)}) with pytest.raises(ValueError, match="source_1"): - ocio.write_nz(s, tmp_path / "short.txt", n_bins=2) + sio.write_nz(s, tmp_path / "short.txt", n_bins=2) diff --git a/src/sp_validation/tests/test_sacc_interop_realdata.py b/src/sp_validation/tests/test_sacc_io_realdata.py similarity index 98% rename from src/sp_validation/tests/test_sacc_interop_realdata.py rename to src/sp_validation/tests/test_sacc_io_realdata.py index df7b1366..abe1f230 100644 --- a/src/sp_validation/tests/test_sacc_interop_realdata.py +++ b/src/sp_validation/tests/test_sacc_io_realdata.py @@ -28,7 +28,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_interop, sacc_io +from sp_validation import sacc_io _DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") _REAL_FILES = { @@ -221,7 +221,7 @@ def test_realdata_roundtrip_byte_equal(label, tmp_path): reference = _current_script_reference(cf, hdul, tmp_path) converted = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -250,7 +250,7 @@ def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) ondisk_names = [h.name for h in hdul] converted = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(converted) as conv: diff --git a/src/sp_validation/tests/test_sacc_interop_twopoint.py b/src/sp_validation/tests/test_sacc_io_twopoint.py similarity index 96% rename from src/sp_validation/tests/test_sacc_interop_twopoint.py rename to src/sp_validation/tests/test_sacc_io_twopoint.py index cd1c4a31..78f6809d 100644 --- a/src/sp_validation/tests/test_sacc_interop_twopoint.py +++ b/src/sp_validation/tests/test_sacc_io_twopoint.py @@ -1,6 +1,6 @@ """Byte-compare tests for the SACC -> 2pt-FITS converter. -The converter (:mod:`sp_validation.sacc_interop`) must reproduce the CosmoSIS +The converter (:mod:`sp_validation.sacc_io`) must reproduce the CosmoSIS 2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau ``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check @@ -36,7 +36,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_interop, sacc_io +from sp_validation import sacc_io _SCRIPT = ( Path(__file__).resolve().parents[3] @@ -290,7 +290,7 @@ def test_plain_xi_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp) s = _sacc(inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -300,7 +300,7 @@ def test_xi_cl_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp, cl=True) s = _sacc(inp, cl=True) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -311,7 +311,7 @@ def test_xi_rho_tau_byte_equal(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) assert out.read_bytes() == reference.read_bytes() @@ -334,7 +334,7 @@ def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(out) as hdul: @@ -351,7 +351,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp = _inputs(seed=0) s = _sacc(inp) out = tmp_path / "base.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) with fits.open(out) as hdul: base_xip = hdul["XI_PLUS"].data["VALUE"].copy() @@ -359,7 +359,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp2["xip"] = inp2["xip"] + 1.0 s2 = _sacc(inp2) out2 = tmp_path / "perturbed.fits" - sacc_interop.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) with fits.open(out2) as hdul: new_xip = hdul["XI_PLUS"].data["VALUE"] @@ -384,7 +384,7 @@ def test_integration_grid_points_ignored(tmp_path): s_plain = _sacc(inp, cl=True, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out_plain = tmp_path / "plain.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -452,7 +452,7 @@ def test_integration_grid_points_ignored(tmp_path): s_aug.add_covariance(full) out_aug = tmp_path / "aug.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -465,7 +465,7 @@ def test_rho_tau_sidecars_required_together(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) with pytest.raises(ValueError, match="together"): - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 ) @@ -490,9 +490,9 @@ def test_tomographic_sacc_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="single-bin only"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) with pytest.raises(ValueError, match="single-bin only"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) assert not (tmp_path / "x.fits").exists() @@ -513,7 +513,7 @@ def test_sacc_without_xi_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="nothing to convert"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) assert not (tmp_path / "x.fits").exists() @@ -538,7 +538,7 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "encoded.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -567,7 +567,7 @@ def _mask_idx(dtype, tracers): _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) - expected = sacc_interop._block_diag( + expected = sacc_io._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) cell_idx = _mask_idx(sacc_io.CL_EE, pair) From 06b24c6f4324f1bfaee7c9281a2992ff55625c86 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 20 Jul 2026 22:30:15 +0200 Subject: [PATCH 37/38] fix(Dockerfile): copy source before the blinding-extra editable install The merged layer order ran the [blinding] editable install before COPY ., so uv-overrides.txt (and the source tree) did not exist in the build context. The blinding install is itself 'uv pip install -e .[blinding]', so the old final --no-deps editable layer is redundant and is dropped. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52038d9f..3caa129f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,11 @@ COPY pyproject.toml uv.lock /sp_validation/ RUN uv sync --frozen --inexact --no-install-project \ --extra test --extra glass --extra workflow +# Full source in place before the blinding-extra install below: it is an +# editable install of the project itself and needs uv-overrides.txt and +# scripts/patch_firecrown.py from the tree. +COPY . /sp_validation + # The [blinding] extra (SACC/Smokescreen blinding stack: firecrown + smokescreen) # is not in uv.lock — firecrown is not on PyPI and declares conda-forge-only / # unused sampler connectors as hard deps, so it needs the override file (see @@ -53,8 +58,3 @@ RUN uv pip install --no-cache-dir 'numpy>=2.2,<2.5' # This patches the installed tree (surgical, pinned-version-checked, loud on # mismatch) and verifies `import firecrown.likelihood; import smokescreen`. RUN python scripts/patch_firecrown.py - -# Install sp_validation itself (editable) into the same venv; deps are already -# satisfied by the sync + blinding-extra install above. -COPY . /sp_validation -RUN uv pip install --no-deps -e . From 67db6764c2f6ea044cf2c693eb3734b2c19faccc Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 20 Jul 2026 22:57:44 +0200 Subject: [PATCH 38/38] fix: reconcile pr7-lineage SACC callers with guarded sacc_io API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The feat/sacc-3 merge brought the guard-era sacc_io whose save() now requires a keyword-only `type={'data','mock'}` provenance tag (PRD #241 §4) and whose load() refuses unblinded `type='data'`. Application code and tests from the pr4/pr7 lineage still called the old `save(s, path)` and referenced a template file the #236 cleanup had relocated, leaving 21 CI failures outside the merge-conflict set. This adapts the callers to the reviewed contract (sacc_io is unchanged). save() type= threading: - CosmologyValidation gains a `sacc_type` ctor kwarg (default "data"), stamped by every part-writer: cosebis/pure_eb/psf_systematics/pseudo_cl now `save(..., type=self.sacc_type)`. - assemble_sacc inherits provenance from its parts: `type=metadata["type"]`. - run_2pcf uses `cv.sacc_type`; run_2pcf_highres stamps "data" (real-catalogue fine covariance). - Synthetic-data tests adopt the reviewed idiom `type="mock"` (loads freely): test_sacc_writers _roundtrip + reload helper, test_assemble_sacc part writer, and the test_pseudo_cl fixture (`sacc_type="mock"`). Relocated FITS engine template: - The #236 folder cleanup moved cosmosis_pipeline_A_ia.ini into cosmosis_config/templates/ (still used by the legacy cosmosis_fitting.py path). The new pr7 Snakemake regime — inference.smk's INFERENCE_TEMPLATE_DIR, generate_inference_config, and its tests — resolves templates from cosmosis_config/ directly, where pr7 placed A_ia_sacc.ini but never copied its FITS sibling. Restore A_ia.ini alongside the sacc template. Fast suite: 254 passed, 1 skipped. Two remaining failures (test_calculate_pure_eb pinned-value drift; test_configured_paths_exist_on_candide) are pre-existing and environmental — they fail identically with these changes stashed, and neither is in the API-drift set. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- .../cosmosis_pipeline_A_ia.ini | 108 ++++++++++++++++++ src/sp_validation/cosmo_val/core.py | 5 + src/sp_validation/cosmo_val/cosebis.py | 2 +- src/sp_validation/cosmo_val/pseudo_cl.py | 2 +- .../cosmo_val/psf_systematics.py | 2 +- src/sp_validation/cosmo_val/pure_eb.py | 2 +- src/sp_validation/tests/test_assemble_sacc.py | 2 +- src/sp_validation/tests/test_pseudo_cl.py | 1 + src/sp_validation/tests/test_sacc_writers.py | 4 +- workflow/scripts/assemble_sacc.py | 3 +- workflow/scripts/run_2pcf.py | 2 +- workflow/scripts/run_2pcf_highres.py | 3 +- 12 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini diff --git a/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini new file mode 100644 index 00000000..eb3ab166 --- /dev/null +++ b/cosmo_inference/cosmosis_config/cosmosis_pipeline_A_ia.ini @@ -0,0 +1,108 @@ +#parameters used elsewhere in this file +[DEFAULT] +COSMOSIS_DIR = /n23data1/n06data/lgoh/scratch/cosmosis-standard-library_lisa + + +[pipeline] +modules = consistency sample_S8 camb load_nz_fits photoz_bias linear_alignment projection add_intrinsic 2pt_shear shear_m_bias 2pt_like +likelihoods = 2pt_like +extra_output = cosmological_parameters/omega_lambda cosmological_parameters/S_8 cosmological_parameters/sigma_8 cosmological_parameters/omega_m +timing = T +debug = T + +[runtime] +sampler = polychord +verbosity = debug + +[polychord] +live_points = 192 +feedback = 3 +resume = T +base_dir = %(SCRATCH)s/polychord + +[test] + +[output] +format = text +lock = F + +[consistency] +file = %(COSMOSIS_DIR)s/utility/consistency/consistency_interface.py +verbose = F + +[sample_S8] +file = %(COSMOSIS_DIR)s/utility/sample_sigma8/sample_S8.py + +[camb] +file = %(COSMOSIS_DIR)s/boltzmann/camb/camb_interface.py +mode=power +lmax=2508 +feedback=0 +do_reionization=F +kmin=1e-5 +kmax=20.0 +nk=200 +zmax=5.0 +zmax_background=5.0 +nz_background=500 +halofit_version=mead2020_feedback +nonlinear=pk +neutrino_hierarchy=normal +kmax_extrapolate = 500.0 + +[load_nz_fits] +file = %(COSMOSIS_DIR)s/number_density/load_nz_fits/load_nz_fits.py +nz_file =%(FITS_FILE)s +data_sets = SOURCE + +[photoz_bias] +file = %(COSMOSIS_DIR)s/number_density/photoz_bias/photoz_bias.py +mode = additive +sample = nz_source +bias_section = nofz_shifts +interpolation = cubic +output_deltaz_section_name = delta_z_out + +[linear_alignment] +file = %(COSMOSIS_DIR)s/intrinsic_alignments/la_model/linear_alignments_interface_znla.py +method = bk_corrected + +[projection] +file = %(COSMOSIS_DIR)s/structure/projection/project_2d.py +ell_min_logspaced = 1.0 +ell_max_logspaced = 25000.0 +n_ell_logspaced = 400 +shear-shear = source-source +shear-intrinsic = source-source +intrinsic-intrinsic = source-source +get_kernel_peaks = F +verbose = F + +[add_intrinsic] +file = %(COSMOSIS_DIR)s/shear/add_intrinsic/add_intrinsic.py +shear-shear=T +position-shear=F +perbin=F + +[2pt_shear] +file = %(COSMOSIS_DIR)s/shear/cl_to_xi_nicaea/nicaea_interface.so +corr_type = 0 ; shear_cl -> shear_xi + +[shear_m_bias] +file = %(COSMOSIS_DIR)s/shear/shear_bias/shear_m_bias.py +m_per_bin = True +; Despite the parameter name, this can operate on xi as well as C_ell. +cl_section = shear_xi_plus shear_xi_minus +verbose = F + +[2pt_like] +file = %(COSMOSIS_DIR)s/likelihood/2pt/2pt_like.py +data_file=%(FITS_FILE)s +gaussian_covariance=F +covmat_name=COVMAT +cut_zeros=F +data_sets=XI_PLUS XI_MINUS +like_name=2pt_like + +angle_range_XI_PLUS_1_1= 10.0 200.0 +angle_range_XI_MINUS_1_1= 20.0 200.0 \ No newline at end of file diff --git a/src/sp_validation/cosmo_val/core.py b/src/sp_validation/cosmo_val/core.py index 0c9273d8..b916e3f8 100644 --- a/src/sp_validation/cosmo_val/core.py +++ b/src/sp_validation/cosmo_val/core.py @@ -224,6 +224,7 @@ def __init__( path_onecovariance=None, cosmo_params=None, blind=None, + sacc_type="data", ): self.rho_tau_method = rho_tau_method self.cov_estimate_method = cov_estimate_method @@ -253,6 +254,10 @@ def __init__( self.nside_mask = nside_mask self.path_onecovariance = path_onecovariance self.blind = blind + # SACC provenance stamped by every part-writer via sacc_io.save(type=…): + # 'data' for real catalogues (load-gated until blinded), 'mock' for + # simulations (freely inspectable). PRD #241 §4. + self.sacc_type = sacc_type assert self.cell_method in ["map", "catalog"], ( "cell_method must be 'map' or 'catalog'" diff --git a/src/sp_validation/cosmo_val/cosebis.py b/src/sp_validation/cosmo_val/cosebis.py index 6b1d6146..9e606e07 100644 --- a/src/sp_validation/cosmo_val/cosebis.py +++ b/src/sp_validation/cosmo_val/cosebis.py @@ -180,7 +180,7 @@ def cosebis_to_sacc_part(self, version, out_path, results, fiducial_scale_cut=No result, scale_cut, ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=self.sacc_type) def plot_cosebis( self, diff --git a/src/sp_validation/cosmo_val/pseudo_cl.py b/src/sp_validation/cosmo_val/pseudo_cl.py index 29ff7846..acffbdfe 100644 --- a/src/sp_validation/cosmo_val/pseudo_cl.py +++ b/src/sp_validation/cosmo_val/pseudo_cl.py @@ -704,7 +704,7 @@ def pseudo_cl_to_sacc_part(self, version, out_path, ell_eff, cl_all, wsp): cl_all, wsp, ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=self.sacc_type) def plot_pseudo_cl(self): """ diff --git a/src/sp_validation/cosmo_val/psf_systematics.py b/src/sp_validation/cosmo_val/psf_systematics.py index af819977..d9900de6 100644 --- a/src/sp_validation/cosmo_val/psf_systematics.py +++ b/src/sp_validation/cosmo_val/psf_systematics.py @@ -80,7 +80,7 @@ def rho_tau_to_sacc_part( tau_cov_th=tau_cov_th, ) out_path = os.path.join(out_dir, f"rho_tau_{base}.sacc") - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=self.sacc_type) @property def rho_stat_handler(self): diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 416ad8ef..37814dbf 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -151,7 +151,7 @@ def pure_eb_to_sacc_part(self, version, out_path, results): eb, covariance=results["cov"], ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=self.sacc_type) def plot_pure_eb( self, diff --git a/src/sp_validation/tests/test_assemble_sacc.py b/src/sp_validation/tests/test_assemble_sacc.py index ccddab97..6b4a9a7f 100644 --- a/src/sp_validation/tests/test_assemble_sacc.py +++ b/src/sp_validation/tests/test_assemble_sacc.py @@ -131,7 +131,7 @@ def get_bandpower_windows(self): paths = {} for name, part in parts.items(): p = tmp_path / f"{name}.sacc" - sio.save(part, str(p)) + sio.save(part, str(p), type="mock") paths[name] = str(p) return paths diff --git a/src/sp_validation/tests/test_pseudo_cl.py b/src/sp_validation/tests/test_pseudo_cl.py index 637b10c2..e9079bfc 100644 --- a/src/sp_validation/tests/test_pseudo_cl.py +++ b/src/sp_validation/tests/test_pseudo_cl.py @@ -169,6 +169,7 @@ def cv(tmp_path): power=0.5, n_ell_bins=N_ELL_BINS, pol_factor=True, + sacc_type="mock", ) cv._test_version = version return cv diff --git a/src/sp_validation/tests/test_sacc_writers.py b/src/sp_validation/tests/test_sacc_writers.py index d886fa0a..82080869 100644 --- a/src/sp_validation/tests/test_sacc_writers.py +++ b/src/sp_validation/tests/test_sacc_writers.py @@ -31,7 +31,7 @@ def _theta(n=6): def _roundtrip(s, tmp_path, name): p = tmp_path / f"{name}.sacc" - sio.save(s, str(p)) + sio.save(s, str(p), type="mock") return sio.load(str(p)) @@ -306,7 +306,7 @@ def test_assemble_from_reloaded_parts(tmp_path): parts = _make_parts(nz) reloaded = [] for i, part in enumerate(parts): - sio.save(part, str(tmp_path / f"part{i}.sacc")) + sio.save(part, str(tmp_path / f"part{i}.sacc"), type="mock") reloaded.append(sio.load(str(tmp_path / f"part{i}.sacc"))) s = sw.assemble_analysis_sacc(nz, META, reloaded) assert type(s.covariance).__name__ == "FullCovariance" diff --git a/workflow/scripts/assemble_sacc.py b/workflow/scripts/assemble_sacc.py index 513ec976..1398d87b 100644 --- a/workflow/scripts/assemble_sacc.py +++ b/workflow/scripts/assemble_sacc.py @@ -167,7 +167,8 @@ def assemble_sacc( if not parts: raise ValueError(f"no parts found for {version}: {part_paths}") s = assemble_analysis_sacc(nz, metadata, parts) - sacc_io.save(s, out_path) + # Provenance is inherited from the parts (all same version → same type). + sacc_io.save(s, out_path, type=metadata["type"]) print(f"Assembled {len(parts)} parts -> {out_path}") return s diff --git a/workflow/scripts/run_2pcf.py b/workflow/scripts/run_2pcf.py index a773d039..5f223b4c 100644 --- a/workflow/scripts/run_2pcf.py +++ b/workflow/scripts/run_2pcf.py @@ -85,7 +85,7 @@ def run_2pcf( out_path = sacc_out or os.path.join( output_dir or cv.cc["paths"]["output"], f"{ver}_xi_coarse.sacc" ) - sacc_io.save(s, out_path) + sacc_io.save(s, out_path, type=cv.sacc_type) print(f"Wrote coarse ξ± SACC part: {out_path}") return gg diff --git a/workflow/scripts/run_2pcf_highres.py b/workflow/scripts/run_2pcf_highres.py index ed3660bf..681e12d8 100644 --- a/workflow/scripts/run_2pcf_highres.py +++ b/workflow/scripts/run_2pcf_highres.py @@ -229,7 +229,8 @@ def write_xi_fine_sacc(gg): variances=np.concatenate([gg.varxip, gg.varxim]), ) out_path = os.path.join(OUTPUT_DIR, f"{VERSION}_xi_fine.sacc") - sacc_io.save(s, out_path) + # Fine ξ± is computed from the real catalogue for the high-res covariance. + sacc_io.save(s, out_path, type="data") log(f" Wrote {out_path}")