From be3c7e63ee83603aac67795b32355998e3002f80 Mon Sep 17 00:00:00 2001 From: Martyn Garcia Date: Mon, 9 Nov 2026 22:26:03 -0700 Subject: [PATCH] fix(packaging,misgan,lint): shrink PyPI sdist, mask observed NaNs before subtraction, and harden lint runner --- .github/workflows/tests.yml | 16 +++++- MANIFEST.in | 20 ++++++++ README.md | 12 +++++ particlegan/__init__.py | 9 +++- particlegan/masked_mixture.py | 3 +- scripts/check_lint.py | 15 +++++- tests/test_lint_baseline.py | 19 ++++++- tests/test_misgan_component_research.py | 4 +- tests/test_public_package_imports.py | 68 ++++++++++++++++++++++++- 9 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 MANIFEST.in diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5ff5d51ed..95295ac26 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -275,7 +275,9 @@ jobs: import tarfile import zipfile + max_bytes = 600_000 for wheel in Path('dist').glob('*.whl'): + assert wheel.stat().st_size < max_bytes, wheel.stat().st_size with zipfile.ZipFile(wheel) as archive: names = archive.namelist() roots = {name.split('/')[0] for name in names} @@ -288,14 +290,24 @@ jobs: # particlegan_forge is the documented external tier API (EXPERIMENTATION.md). packages = {'particlegan', 'particlegan_forge'} assert all(root in packages or root.endswith('.dist-info') for root in roots), roots - research = {'reports', 'results', 'experiments', 'benchmarks', 'lib', 'configs', 'data', 'runs', 'artifacts'} + excluded = { + 'tests', 'reports', 'results', 'experiments', 'benchmarks', 'lib', + 'configs', 'data', 'runs', 'artifacts', 'docs', 'examples', 'ops', 'scripts', + } + allowed_sdist_roots = { + 'LICENSE', 'MANIFEST.in', 'PKG-INFO', 'README.md', + 'particlegan', 'particlegan.egg-info', 'particlegan_forge', + 'pyproject.toml', 'setup.cfg', + } for source in Path('dist').glob('*.tar.gz'): + assert source.stat().st_size < max_bytes, source.stat().st_size with tarfile.open(source) as archive: names = archive.getnames() roots = {parts[1] for name in names if len(parts := name.split('/')) > 1} license_path = next(name for name in names if name.count('/') == 1 and name.endswith('/LICENSE')) assert archive.extractfile(license_path).read() == Path('LICENSE').read_bytes() - assert not roots & research, roots & research + assert not roots & excluded, roots & excluded + assert roots <= allowed_sdist_roots, roots - allowed_sdist_roots PY - name: Validate package metadata and README run: python -m twine check --strict dist/* diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..a2bedd00a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,20 @@ +prune tests +prune reports +prune results +prune experiments +prune benchmarks +prune lib +prune configs +prune data +prune runs +prune artifacts +prune docs +prune examples +prune ops +prune scripts +exclude .ruff-baseline.json +exclude AGENTS.md +exclude CLAUDE.md +exclude EXPERIMENTATION.md +exclude RESEARCH_GLOSSARY.md +exclude TODO.md diff --git a/README.md b/README.md index efaade5b3..c8600dd4e 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,17 @@ What the evidence does and does not show today: gaps are inside the run's own checkpoint-to-checkpoint and rerun noise (up to 9.4 FID5k between same-config reruns), so no image-scale prior effect is established. +- **BISPF shape/step-gate ablation (`bispf_shape_gates=False`).** In the same + [2026-10-25 ablation study](reports/prior-ablation-2026-10-25/README.md) (Arm A4, + `bispf-pure-global-v1`), disabling all 10 shape/step gates in `BISPFGates` drops + full-suite passes from **19/49 to 13/49** (and from **7/15 to 5/15** on the matched + sampled-latent subset), losing `ring16_acquisition` (reaches `hq = 0.958` at the + endpoint but only 2/5 consecutive stable checks), `mode_hold` (7/8 modes), + `ring_extension` and `img_stripes2`. For wide continuous vectors (`dim >= 170`) where + only the automatic categorical simplex heuristic is unwanted, pass + `penalty_options={"input_domain": "continuous"}` to `GANTrainer` (or + `recipe.make_critic_penalty(opt_d, input_domain="continuous")`) as documented in + [BISPF input domains](docs/bispf-input-domains.md). - **Headline numbers below are single runs** without error bars, unless a linked report says otherwise. @@ -245,6 +256,7 @@ transport their parent's Adam history at half mass. ## Learn more - [Optimized BISPF settings and measured behavior](reports/forge/bispf-optimized/README.md) +- [BISPF continuous vs. categorical input domains](docs/bispf-input-domains.md): explicit penalty domain declarations (`input_domain="continuous"` or `"categorical"`) for wide continuous vectors (`dim >= 170`) and categorical codebooks - [Historical KA2 formulation](docs/ka2.md), including several critics and conditional critics - [E22](docs/e22.md): a schedule-free configuration for the native 100-Gaussian problems with no data-space statistics - [ParticleGAN Atlas](docs/atlas.md): illustrated E22 comparison, local population corrections and automatic backend selection diff --git a/particlegan/__init__.py b/particlegan/__init__.py index a90cd2c60..539ce6831 100644 --- a/particlegan/__init__.py +++ b/particlegan/__init__.py @@ -16,6 +16,12 @@ from .diffusion import DDGAN from .discriminators import BatchDistanceDiscriminator, LinearSkipDiscriminator from .gan_loss import GANLoss +from .joint_reconstruction import joint_reconstruction_loss +from .masked_mixture import ( + NeuralComponentImputer, + masked_mixture_nll, + masked_quantile_transport_loss, +) from .particle_prior import GaussianPrior, MoGParticlePrior, ParticlePrior, calibrate_mog_sigma from .priors import ( AdditiveNormalManipulator, @@ -73,7 +79,8 @@ "StratifiedBlueNoisePrior", "SBNPPriorRegularizer", "ContinuousThermalCooler", "BISPFEngine", "BISPFLossAdapter", "BISPFStepRecord", "BISPFCriticAdam", "BISPFCriticPenalty", "SinkhornRelativisticLoss", "BISPFGates", "DDGAN", "UCD", "ucd_labels", "ucd_loss", "ucd_scores", - "conditional_energy_score", + "conditional_energy_score", "joint_reconstruction_loss", + "masked_mixture_nll", "NeuralComponentImputer", "masked_quantile_transport_loss", "DEFAULT_RECIPE", "Recipe", "NetworkLRTransition", "DataDriftController", "get_recipe", "get_recipe_fields", "learning_rate_scale", "learning_rate_scales", "scale_learning_rates", "GANTrainer", "InputNoise", "BatchDistanceDiscriminator", "LinearSkipDiscriminator", ] diff --git a/particlegan/masked_mixture.py b/particlegan/masked_mixture.py index 4f2a361e1..468c13a6e 100644 --- a/particlegan/masked_mixture.py +++ b/particlegan/masked_mixture.py @@ -23,7 +23,8 @@ def masked_mixture_nll(observed, mask, components, *, sigma): raise ValueError("sigma must be finite and positive") if not bool(((mask == 0) | (mask == 1)).all()) or bool((mask.sum(1) == 0).any()): raise ValueError("mask must be binary with at least one observed sensor per row") - residual = torch.where(mask[:, None].bool(), observed[:, None] - components[None], 0.) + clean = torch.where(mask.bool(), observed, 0.) + residual = torch.where(mask[:, None].bool(), clean[:, None] - components[None], 0.) count = mask.sum(1) logp = -.5 * residual.square().sum(-1) / sigma**2 - count[:, None] * math.log(sigma * math.sqrt(2 * math.pi)) return ((math.log(len(components)) - torch.logsumexp(logp, dim=1)) / count).mean() diff --git a/scripts/check_lint.py b/scripts/check_lint.py index 819c946dd..21557d2ed 100644 --- a/scripts/check_lint.py +++ b/scripts/check_lint.py @@ -8,9 +8,11 @@ import argparse from collections import Counter import hashlib +import importlib.util import json from pathlib import Path import re +import shutil import subprocess import sys @@ -40,13 +42,22 @@ } +def _ruff_command() -> list[str]: + if importlib.util.find_spec("ruff") is not None: + return [sys.executable, "-m", "ruff"] + binary = shutil.which("ruff") + if binary: + return [binary] + return [sys.executable, "-m", "ruff"] + + def findings(root: Path) -> Counter: result = subprocess.run( - [sys.executable, "-m", "ruff", "check", "--select", RULES, + [*_ruff_command(), "check", "--select", RULES, "--output-format", "json", *PATHS], cwd=root, capture_output=True, text=True, ) - if result.returncode not in (0, 1): + if result.returncode not in (0, 1) or not result.stdout.strip(): raise RuntimeError(result.stderr.strip() or "Ruff failed") counts = Counter() for item in json.loads(result.stdout): diff --git a/tests/test_lint_baseline.py b/tests/test_lint_baseline.py index 02507fe58..12f87b9b2 100644 --- a/tests/test_lint_baseline.py +++ b/tests/test_lint_baseline.py @@ -1,8 +1,12 @@ """Exercise the actual lint gate without changing repository sources.""" import json from pathlib import Path +import subprocess -from scripts.check_lint import FROZEN_F821_FALSE_POSITIVES, PATHS, main +import pytest + +import scripts.check_lint as check_lint +from scripts.check_lint import FROZEN_F821_FALSE_POSITIVES, PATHS, findings, main def _workspace(root): @@ -77,3 +81,16 @@ def test_frozen_false_positive_exception_cannot_hide_a_changed_source(tmp_path, assert main([*args, "--update-baseline"]) == 1 assert "new_undefined_name" in capsys.readouterr().out assert (tmp_path / ".ruff-baseline.json").read_bytes() == baseline + + +def test_missing_ruff_or_empty_output_raises_clear_runtime_error(tmp_path, monkeypatch): + _workspace(tmp_path) + monkeypatch.setattr( + check_lint.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args=args[0], returncode=1, stdout="", stderr="No module named ruff\n" + ), + ) + with pytest.raises(RuntimeError, match="No module named ruff"): + findings(tmp_path) diff --git a/tests/test_misgan_component_research.py b/tests/test_misgan_component_research.py index 36e3563a9..ef4e1a162 100644 --- a/tests/test_misgan_component_research.py +++ b/tests/test_misgan_component_research.py @@ -56,12 +56,14 @@ def test_scorer_oracle_and_uncertainty_controls(): def test_observed_likelihood_ignores_missing_values_and_detaches_teacher(): - x=torch.tensor([[1.,float('nan')],[1.,999.]]) + x=torch.tensor([[1.,float('nan')],[1.,999.]],requires_grad=True) mask=torch.tensor([[1.,0.],[1.,0.]]) centers=torch.tensor([[0.,2.],[1.,3.]],requires_grad=True) loss=masked_mixture_nll(x,mask,centers,sigma=.1) loss.backward() assert torch.isfinite(loss) + assert torch.isfinite(x.grad).all() + assert torch.equal(x.grad[:,1],torch.zeros(2)) assert torch.equal(centers.grad[:,1],torch.zeros(2)) net=NeuralComponentImputer(2,2) teacher=torch.randn(4,2,requires_grad=True) diff --git a/tests/test_public_package_imports.py b/tests/test_public_package_imports.py index 2d392c0ed..0b5514fc3 100644 --- a/tests/test_public_package_imports.py +++ b/tests/test_public_package_imports.py @@ -30,12 +30,23 @@ def find_spec(self, fullname, path=None, target=None): raise AssertionError('core import requested ' + fullname) sys.meta_path.insert(0, NoPlotting()) - from particlegan import ManipulatedPrior, ParticlePrior + from particlegan import ( + ManipulatedPrior, + NeuralComponentImputer, + ParticlePrior, + joint_reconstruction_loss, + masked_mixture_nll, + masked_quantile_transport_loss, + ) from particlegan_forge import NamedStreams, TaskSpec, render_training_gif from particlegan_forge.__main__ import main assert get_type_hints(ParticlePrior.manipulate)['return'] is ManipulatedPrior assert ParticlePrior(num_particles=2).sample(1)[0].shape == (1, 4) + assert all(callable(fn) for fn in ( + joint_reconstruction_loss, masked_mixture_nll, + masked_quantile_transport_loss, NeuralComponentImputer, + )) assert NamedStreams().generator('data').device.type == 'cpu' assert callable(render_training_gif) and callable(main) and TaskSpec assert not any(name.split('.')[0] in {'matplotlib', 'PIL'} for name in sys.modules) @@ -97,3 +108,58 @@ def test_renderer_preserves_matplotlib_backend_and_writes_observed_frames(tmp_pa assert matplotlib.get_backend() == before_backend assert plt.get_fignums() == before_figures """, tmp_path / "fixture.gif") + + +def test_pypi_wheel_and_sdist_remain_tiny_and_exclude_non_package_trees(tmp_path): + _python(""" + from pathlib import Path + import shutil + import sys + import tarfile + import zipfile + import setuptools.build_meta as build_meta + + root = Path.cwd() + stage = Path(sys.argv[1]) / "stage" + dist = Path(sys.argv[1]) / "dist" + stage.mkdir() + dist.mkdir() + for name in ("pyproject.toml", "MANIFEST.in", "README.md", "LICENSE"): + shutil.copy2(root / name, stage / name) + for pkg in ("particlegan", "particlegan_forge"): + shutil.copytree(root / pkg, stage / pkg) + for excluded in ( + "tests", "reports", "results", "experiments", "benchmarks", "lib", + "configs", "data", "runs", "artifacts", "docs", "examples", "ops", "scripts", + ): + (stage / excluded).mkdir() + (stage / excluded / "probe.py").write_text("raise RuntimeError('should not be packaged')\\n") + + import os + os.chdir(stage) + wheel = dist / build_meta.build_wheel(str(dist)) + sdist = dist / build_meta.build_sdist(str(dist)) + + max_bytes = 600_000 + assert wheel.stat().st_size < max_bytes, wheel.stat().st_size + assert sdist.stat().st_size < max_bytes, sdist.stat().st_size + + with zipfile.ZipFile(wheel) as archive: + wheel_roots = {name.split("/")[0] for name in archive.namelist()} + assert all( + r in {"particlegan", "particlegan_forge"} or r.endswith(".dist-info") + for r in wheel_roots + ), wheel_roots + + allowed_sdist_roots = { + "LICENSE", "MANIFEST.in", "PKG-INFO", "README.md", + "particlegan", "particlegan.egg-info", "particlegan_forge", + "pyproject.toml", "setup.cfg", + } + with tarfile.open(sdist) as archive: + sdist_roots = { + parts[1] for name in archive.getnames() + if len(parts := name.split("/")) > 1 + } + assert sdist_roots <= allowed_sdist_roots, sdist_roots - allowed_sdist_roots + """, tmp_path)