Shrink PyPI sdist, mask observed NaNs before subtraction, and harden lint runner

.github/workflows/tests.yml

@@ -275,7 +275,9 @@ jobs:
           import tarfile
           import zipfile
 
+          max_bytes = 600_000
           for wheel in Path('dist').glob('*.whl'):
+              assert wheel.stat().st_size < max_bytes, wheel.stat().st_size
               with zipfile.ZipFile(wheel) as archive:
                   names = archive.namelist()
                   roots = {name.split('/')[0] for name in names}
@@ -288,14 +290,24 @@ jobs:
               # particlegan_forge is the documented external tier API (EXPERIMENTATION.md).
               packages = {'particlegan', 'particlegan_forge'}
               assert all(root in packages or root.endswith('.dist-info') for root in roots), roots
-          research = {'reports', 'results', 'experiments', 'benchmarks', 'lib', 'configs', 'data', 'runs', 'artifacts'}
+          excluded = {
+              'tests', 'reports', 'results', 'experiments', 'benchmarks', 'lib',
+              'configs', 'data', 'runs', 'artifacts', 'docs', 'examples', 'ops', 'scripts',
+          }
+          allowed_sdist_roots = {
+              'LICENSE', 'MANIFEST.in', 'PKG-INFO', 'README.md',
+              'particlegan', 'particlegan.egg-info', 'particlegan_forge',
+              'pyproject.toml', 'setup.cfg',
+          }
           for source in Path('dist').glob('*.tar.gz'):
+              assert source.stat().st_size < max_bytes, source.stat().st_size
               with tarfile.open(source) as archive:
                   names = archive.getnames()
                   roots = {parts[1] for name in names if len(parts := name.split('/')) > 1}
                   license_path = next(name for name in names if name.count('/') == 1 and name.endswith('/LICENSE'))
                   assert archive.extractfile(license_path).read() == Path('LICENSE').read_bytes()
-              assert not roots & research, roots & research
+              assert not roots & excluded, roots & excluded
+              assert roots <= allowed_sdist_roots, roots - allowed_sdist_roots
           PY
       - name: Validate package metadata and README
         run: python -m twine check --strict dist/*

MANIFEST.in

@@ -0,0 +1,20 @@
+prune tests
+prune reports
+prune results
+prune experiments
+prune benchmarks
+prune lib
+prune configs
+prune data
+prune runs
+prune artifacts
+prune docs
+prune examples
+prune ops
+prune scripts
+exclude .ruff-baseline.json
+exclude AGENTS.md
+exclude CLAUDE.md
+exclude EXPERIMENTATION.md
+exclude RESEARCH_GLOSSARY.md
+exclude TODO.md

README.md

@@ -128,6 +128,17 @@ What the evidence does and does not show today:
   gaps are inside the run's own checkpoint-to-checkpoint and rerun noise
   (up to 9.4 FID5k between same-config reruns), so no image-scale prior effect is
   established.
+- **BISPF shape/step-gate ablation (`bispf_shape_gates=False`).** In the same
+  [2026-10-25 ablation study](reports/prior-ablation-2026-10-25/README.md) (Arm A4,
+  `bispf-pure-global-v1`), disabling all 10 shape/step gates in `BISPFGates` drops
+  full-suite passes from **19/49 to 13/49** (and from **7/15 to 5/15** on the matched
+  sampled-latent subset), losing `ring16_acquisition` (reaches `hq = 0.958` at the
+  endpoint but only 2/5 consecutive stable checks), `mode_hold` (7/8 modes),
+  `ring_extension` and `img_stripes2`. For wide continuous vectors (`dim >= 170`) where
+  only the automatic categorical simplex heuristic is unwanted, pass
+  `penalty_options={"input_domain": "continuous"}` to `GANTrainer` (or
+  `recipe.make_critic_penalty(opt_d, input_domain="continuous")`) as documented in
+  [BISPF input domains](docs/bispf-input-domains.md).
 - **Headline numbers below are single runs** without error bars, unless a linked
   report says otherwise.
 
@@ -245,6 +256,7 @@ transport their parent's Adam history at half mass.
 ## Learn more
 
 - [Optimized BISPF settings and measured behavior](reports/forge/bispf-optimized/README.md)
+- [BISPF continuous vs. categorical input domains](docs/bispf-input-domains.md): explicit penalty domain declarations (`input_domain="continuous"` or `"categorical"`) for wide continuous vectors (`dim >= 170`) and categorical codebooks
 - [Historical KA2 formulation](docs/ka2.md), including several critics and conditional critics
 - [E22](docs/e22.md): a schedule-free configuration for the native 100-Gaussian problems with no data-space statistics
 - [ParticleGAN Atlas](docs/atlas.md): illustrated E22 comparison, local population corrections and automatic backend selection

particlegan/__init__.py

@@ -16,6 +16,12 @@
 from .diffusion import DDGAN
 from .discriminators import BatchDistanceDiscriminator, LinearSkipDiscriminator
 from .gan_loss import GANLoss
+from .joint_reconstruction import joint_reconstruction_loss
+from .masked_mixture import (
+    NeuralComponentImputer,
+    masked_mixture_nll,
+    masked_quantile_transport_loss,
+)
 from .particle_prior import GaussianPrior, MoGParticlePrior, ParticlePrior, calibrate_mog_sigma
 from .priors import (
     AdditiveNormalManipulator,
@@ -73,7 +79,8 @@
     "StratifiedBlueNoisePrior", "SBNPPriorRegularizer", "ContinuousThermalCooler", "BISPFEngine", "BISPFLossAdapter",
     "BISPFStepRecord", "BISPFCriticAdam", "BISPFCriticPenalty", "SinkhornRelativisticLoss", "BISPFGates",
     "DDGAN", "UCD", "ucd_labels", "ucd_loss", "ucd_scores",
-    "conditional_energy_score",
+    "conditional_energy_score", "joint_reconstruction_loss",
+    "masked_mixture_nll", "NeuralComponentImputer", "masked_quantile_transport_loss",
     "DEFAULT_RECIPE", "Recipe", "NetworkLRTransition", "DataDriftController", "get_recipe", "get_recipe_fields", "learning_rate_scale", "learning_rate_scales", "scale_learning_rates", "GANTrainer", "InputNoise",
     "BatchDistanceDiscriminator", "LinearSkipDiscriminator",
 ]

particlegan/masked_mixture.py

@@ -23,7 +23,8 @@ def masked_mixture_nll(observed, mask, components, *, sigma):
         raise ValueError("sigma must be finite and positive")
     if not bool(((mask == 0) | (mask == 1)).all()) or bool((mask.sum(1) == 0).any()):
         raise ValueError("mask must be binary with at least one observed sensor per row")
-    residual = torch.where(mask[:, None].bool(), observed[:, None] - components[None], 0.)
+    clean = torch.where(mask.bool(), observed, 0.)
+    residual = torch.where(mask[:, None].bool(), clean[:, None] - components[None], 0.)
     count = mask.sum(1)
     logp = -.5 * residual.square().sum(-1) / sigma**2 - count[:, None] * math.log(sigma * math.sqrt(2 * math.pi))
     return ((math.log(len(components)) - torch.logsumexp(logp, dim=1)) / count).mean()

scripts/check_lint.py

@@ -8,9 +8,11 @@
 import argparse
 from collections import Counter
 import hashlib
+import importlib.util
 import json
 from pathlib import Path
 import re
+import shutil
 import subprocess
 import sys
 
@@ -40,13 +42,22 @@
 }
 
 
+def _ruff_command() -> list[str]:
+    if importlib.util.find_spec("ruff") is not None:
+        return [sys.executable, "-m", "ruff"]
+    binary = shutil.which("ruff")
+    if binary:
+        return [binary]
+    return [sys.executable, "-m", "ruff"]
+
+
 def findings(root: Path) -> Counter:
     result = subprocess.run(
-        [sys.executable, "-m", "ruff", "check", "--select", RULES,
+        [*_ruff_command(), "check", "--select", RULES,
          "--output-format", "json", *PATHS],
         cwd=root, capture_output=True, text=True,
     )
-    if result.returncode not in (0, 1):
+    if result.returncode not in (0, 1) or not result.stdout.strip():
         raise RuntimeError(result.stderr.strip() or "Ruff failed")
     counts = Counter()
     for item in json.loads(result.stdout):

tests/test_lint_baseline.py

@@ -1,8 +1,12 @@
 """Exercise the actual lint gate without changing repository sources."""
 import json
 from pathlib import Path
+import subprocess
 
-from scripts.check_lint import FROZEN_F821_FALSE_POSITIVES, PATHS, main
+import pytest
+
+import scripts.check_lint as check_lint
+from scripts.check_lint import FROZEN_F821_FALSE_POSITIVES, PATHS, findings, main
 
 
 def _workspace(root):
@@ -77,3 +81,16 @@ def test_frozen_false_positive_exception_cannot_hide_a_changed_source(tmp_path,
     assert main([*args, "--update-baseline"]) == 1
     assert "new_undefined_name" in capsys.readouterr().out
     assert (tmp_path / ".ruff-baseline.json").read_bytes() == baseline
+
+
+def test_missing_ruff_or_empty_output_raises_clear_runtime_error(tmp_path, monkeypatch):
+    _workspace(tmp_path)
+    monkeypatch.setattr(
+        check_lint.subprocess,
+        "run",
+        lambda *args, **kwargs: subprocess.CompletedProcess(
+            args=args[0], returncode=1, stdout="", stderr="No module named ruff\n"
+        ),
+    )
+    with pytest.raises(RuntimeError, match="No module named ruff"):
+        findings(tmp_path)

tests/test_misgan_component_research.py

@@ -56,12 +56,14 @@ def test_scorer_oracle_and_uncertainty_controls():
 
 
 def test_observed_likelihood_ignores_missing_values_and_detaches_teacher():
-    x=torch.tensor([[1.,float('nan')],[1.,999.]])
+    x=torch.tensor([[1.,float('nan')],[1.,999.]],requires_grad=True)
     mask=torch.tensor([[1.,0.],[1.,0.]])
     centers=torch.tensor([[0.,2.],[1.,3.]],requires_grad=True)
     loss=masked_mixture_nll(x,mask,centers,sigma=.1)
     loss.backward()
     assert torch.isfinite(loss)
+    assert torch.isfinite(x.grad).all()
+    assert torch.equal(x.grad[:,1],torch.zeros(2))
     assert torch.equal(centers.grad[:,1],torch.zeros(2))
     net=NeuralComponentImputer(2,2)
     teacher=torch.randn(4,2,requires_grad=True)

tests/test_public_package_imports.py

@@ -30,12 +30,23 @@ def find_spec(self, fullname, path=None, target=None):
                     raise AssertionError('core import requested ' + fullname)
 
         sys.meta_path.insert(0, NoPlotting())
-        from particlegan import ManipulatedPrior, ParticlePrior
+        from particlegan import (
+            ManipulatedPrior,
+            NeuralComponentImputer,
+            ParticlePrior,
+            joint_reconstruction_loss,
+            masked_mixture_nll,
+            masked_quantile_transport_loss,
+        )
         from particlegan_forge import NamedStreams, TaskSpec, render_training_gif
         from particlegan_forge.__main__ import main
 
         assert get_type_hints(ParticlePrior.manipulate)['return'] is ManipulatedPrior
         assert ParticlePrior(num_particles=2).sample(1)[0].shape == (1, 4)
+        assert all(callable(fn) for fn in (
+            joint_reconstruction_loss, masked_mixture_nll,
+            masked_quantile_transport_loss, NeuralComponentImputer,
+        ))
         assert NamedStreams().generator('data').device.type == 'cpu'
         assert callable(render_training_gif) and callable(main) and TaskSpec
         assert not any(name.split('.')[0] in {'matplotlib', 'PIL'} for name in sys.modules)
@@ -97,3 +108,58 @@ def test_renderer_preserves_matplotlib_backend_and_writes_observed_frames(tmp_pa
         assert matplotlib.get_backend() == before_backend
         assert plt.get_fignums() == before_figures
     """, tmp_path / "fixture.gif")
+
+
+def test_pypi_wheel_and_sdist_remain_tiny_and_exclude_non_package_trees(tmp_path):
+    _python("""
+        from pathlib import Path
+        import shutil
+        import sys
+        import tarfile
+        import zipfile
+        import setuptools.build_meta as build_meta
+
+        root = Path.cwd()
+        stage = Path(sys.argv[1]) / "stage"
+        dist = Path(sys.argv[1]) / "dist"
+        stage.mkdir()
+        dist.mkdir()
+        for name in ("pyproject.toml", "MANIFEST.in", "README.md", "LICENSE"):
+            shutil.copy2(root / name, stage / name)
+        for pkg in ("particlegan", "particlegan_forge"):
+            shutil.copytree(root / pkg, stage / pkg)
+        for excluded in (
+            "tests", "reports", "results", "experiments", "benchmarks", "lib",
+            "configs", "data", "runs", "artifacts", "docs", "examples", "ops", "scripts",
+        ):
+            (stage / excluded).mkdir()
+            (stage / excluded / "probe.py").write_text("raise RuntimeError('should not be packaged')\\n")
+
+        import os
+        os.chdir(stage)
+        wheel = dist / build_meta.build_wheel(str(dist))
+        sdist = dist / build_meta.build_sdist(str(dist))
+
+        max_bytes = 600_000
+        assert wheel.stat().st_size < max_bytes, wheel.stat().st_size
+        assert sdist.stat().st_size < max_bytes, sdist.stat().st_size
+
+        with zipfile.ZipFile(wheel) as archive:
+            wheel_roots = {name.split("/")[0] for name in archive.namelist()}
+        assert all(
+            r in {"particlegan", "particlegan_forge"} or r.endswith(".dist-info")
+            for r in wheel_roots
+        ), wheel_roots
+
+        allowed_sdist_roots = {
+            "LICENSE", "MANIFEST.in", "PKG-INFO", "README.md",
+            "particlegan", "particlegan.egg-info", "particlegan_forge",
+            "pyproject.toml", "setup.cfg",
+        }
+        with tarfile.open(sdist) as archive:
+            sdist_roots = {
+                parts[1] for name in archive.getnames()
+                if len(parts := name.split("/")) > 1
+            }
+        assert sdist_roots <= allowed_sdist_roots, sdist_roots - allowed_sdist_roots
+    """, tmp_path)