Catch rift_O4d up with oshaughnessy-junior/rift_O4d (integrator/calmarg/convergence payload + CHANGES.rst since rc1)

.github/workflows/ci.yml

@@ -171,6 +171,34 @@ jobs:
       - name: Run simulation_manager smoke test
         run: bash .travis/test-simulation-manager.sh
 
+  integrator-gate-accounting-check:
+    needs: install
+    runs-on: ubuntu-latest
+    # The shape-recovery gate and its opt-in flag probe are far too expensive for CI and stay out
+    # of it. The ACCOUNTING that decides whether one of their rows BLOCKS a merge is neither: it is
+    # pure logic, runs in seconds (no sampler is ever built -- the gate seams are stubbed), and a
+    # bug in it silently CLEARS a real regression. That is not hypothetical: a noisy row failed a
+    # PR, and the confirmation added to absorb it could be configured to re-test nothing, and its
+    # two arms shared a monkey-patched sampler factory so the "default" arm ran with the flag on.
+    # So this part runs on every pipeline.
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: requirements.txt
+      - name: Enable symlink
+        run: sudo ln -sf $(which python3) /usr/bin/python
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip --break-system-packages
+          python -m pip install -r requirements.txt --break-system-packages
+          python -m pip install coverage pytest --break-system-packages
+          python -m pip install --editable . --break-system-packages
+      - name: Run probe confirm-on-fail accounting tests
+        run: python -m pytest -q MonteCarloMarginalizeCode/Code/test/expensive_before_merging/integrators/test_probe_confirm.py
+
   lisa-check:
     needs: install
     runs-on: ubuntu-latest
@@ -192,6 +220,27 @@ jobs:
       - name: Run LISA smoke and contract tests
         run: bash .travis/test-lisa.sh
 
+  calmarg-check:
+    needs: install
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: requirements.txt
+      - name: Enable symlink
+        run: sudo ln -sf $(which python3) /usr/bin/python
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip --break-system-packages
+          python -m pip install -r requirements.txt --break-system-packages
+          python -m pip install coverage pytest --break-system-packages
+          python -m pip install --editable . --break-system-packages
+      - name: Run calibration-marginalization regression gate
+        run: bash .travis/test-calmarg.sh
+
   integration-check:
     needs: install
     runs-on: ubuntu-latest
@@ -287,6 +336,24 @@ jobs:
       - name: Show resolved versions
         run: |
           python -c "import sys, numpy, scipy; print('python', sys.version); print('numpy', numpy.__version__); print('scipy', scipy.__version__)"
+      - name: Run sampler unit/regression tests
+        # Also run by .travis/test-integrate.sh (integration-check, py3.10). Repeated here
+        # because this job is the only one matrixed over BOTH numpy lanes, and these tests
+        # are the kind that break on numpy API removals (e.g. np.trapz -> np.trapezoid).
+        run: python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_limit_cosine_samplers.py
+      - name: Run adaptive-volume collapse / evidence-accounting regressions
+        # These were unguarded.  The AV live-volume-collapse family, the L0 rescue's warm
+        # seed, and the portfolio fair-draw backend each shipped a regression suite that NO
+        # CI job invoked, so a required check could stay green while any of them broke --
+        # including suites from already-merged PRs.  They are exactly the kind that regress
+        # silently: an evidence normalization or an inclusion probability is still a
+        # plausible-looking number when it is wrong.  Fast (seconds each), CPU-only, and
+        # matrixed over both numpy lanes here.
+        run: |
+          python -m pytest -q \
+            MonteCarloMarginalizeCode/Code/test/test_av_empty_live_volume.py \
+            MonteCarloMarginalizeCode/Code/test/test_l0_rescue_seed.py \
+            MonteCarloMarginalizeCode/Code/test/test_portfolio_fairdraw_backend.py
       - name: Run test scripts
         run: |
           . .travis/test-coord.sh
@@ -316,6 +383,10 @@ jobs:
     # so a red run should alert maintainers, not block unrelated PRs.
     runs-on: ubuntu-latest
     continue-on-error: true
+    env:
+      # pygsl_lite 0.1.8 uses the pre-84 distutils.spawn calling convention.
+      # This affects only PEP 517 build backends, not the unpinned runtime solve.
+      PIP_BUILD_CONSTRAINT: ${{ github.workspace }}/containers/constraints-container-build.txt
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5

.gitlab-ci.yml

@@ -106,6 +106,22 @@ sim_manager_check:
     - python -m pip install lscsoft-glue --break-system-packages || echo "lscsoft-glue unavailable; condor portion will skip"
     - bash .travis/test-simulation-manager.sh
 
+calmarg_check:
+  stage: unit tests
+  script:
+    # in-loop calibration marginalization + per-realization self-term (CPU).
+    - bash .travis/test-calmarg.sh
+
+integrator_gate_accounting_check:
+  stage: unit tests
+  script:
+    # The shape-recovery gate and its opt-in flag probe are far too expensive for CI and stay out
+    # of it.  The ACCOUNTING that decides whether one of their rows blocks a merge is neither: it
+    # is pure logic, runs in seconds (no sampler is built), and a bug in it silently CLEARS a real
+    # regression -- which is how a noisy row failed a PR and how an untested row could pass one.
+    # So that part runs on every pipeline.
+    - python -m pytest -q MonteCarloMarginalizeCode/Code/test/expensive_before_merging/integrators/test_probe_confirm.py
+
 test_run:
   stage: system tests
   script:
@@ -140,7 +156,7 @@ gpu_integration:
       --env GW_SURROGATE="$GW_SURROGATE"
       --env RIFT_CI_REQUIRE_GPU="$RIFT_CI_REQUIRE_GPU"
       "$RIFT_CI_APPTAINER_IMAGE"
-      bash -lc 'cd "$CI_PROJECT_DIR" && export PYTHONPATH="$CI_PROJECT_DIR/MonteCarloMarginalizeCode/Code${PYTHONPATH:+:$PYTHONPATH}" && bash .travis/test-integrate.sh'
+      bash -lc 'cd "$CI_PROJECT_DIR" && export PYTHONPATH="$CI_PROJECT_DIR/MonteCarloMarginalizeCode/Code${PYTHONPATH:+:$PYTHONPATH}" && bash .travis/test-integrate.sh && bash .travis/test-calmarg-gpu.sh'
   rules:
     - if: '$CI_PIPELINE_SOURCE == "web"'
       when: manual

.travis/test-build.sh

@@ -61,4 +61,39 @@ assert_has    `pwd`/test_build_slices/ILE.sub      "--distance-marginalization "
 assert_absent `pwd`/test_build_slices/ILE_extr.sub "--distance-marginalization "
 echo "OK: Plan-B slice export only on ILE_extr.sub; distance marginalization disabled only at the extrinsic stage"
 
+# --- 4. Gauss-early ('G') CIP groups use the alternate exe where a real CIP runs ---
+# --use-gauss-early makes the first cip-args-list entry a G group.  Two regimes:
+#
+# 4a. Non-flat exploded jobs (gp fit method): the master job runs a real
+#     CIP (it saves the shared fit), so the G exe must land in the master sub
+#     (CIP_0.sub) and not just the worker sub; non-G groups keep the standard
+#     CIP.  Without explode jobs the master is the only CIP, same assignment.
+#     The ini's [rift-pseudo-pipe] section overrides the command line, so the
+#     gp fit method must be set in an ini variant, not via --cip-fit-method.
+GAUSS_INI=`pwd`/test_build_gauss_gpfit.ini
+sed 's/^cip-fit-method=.*/cip-fit-method="gp"/' $REF_INI > $GAUSS_INI
+util_RIFT_pseudo_pipe.py --use-ini $GAUSS_INI --use-coinc $COINC --use-rundir `pwd`/test_build_gauss --fake-data-cache `pwd`/foo.cache --use-gauss-early
+assert_has    `pwd`/test_build_gauss/CIP_0.sub        "GaussianResampling"
+assert_absent `pwd`/test_build_gauss/CIP_0.sub        "GenericCoordinates"
+assert_has    `pwd`/test_build_gauss/CIP_worker0.sub  "GaussianResampling"
+assert_has    `pwd`/test_build_gauss/CIP_1.sub        "GenericCoordinates"
+assert_absent `pwd`/test_build_gauss/CIP_1.sub        "GaussianResampling"
+echo "OK: gauss-early G group uses GaussianResampling in master and worker subs (non-flat)"
+
+# 4b. Flat exploded jobs (default here: non-gp fit method): no shared fit, so
+#     the master is a documented /bin/true no-op for EVERY group -- including G
+#     groups, whose exe must appear only in the worker sub.  Matches the
+#     non-per-iteration CIP.sub, which is also /bin/true in flat mode.
+util_RIFT_pseudo_pipe.py --use-ini $REF_INI --use-coinc $COINC --use-rundir `pwd`/test_build_gauss_flat --fake-data-cache `pwd`/foo.cache --use-gauss-early
+# CIP.sub (the non-per-iteration master) goes /bin/true via long-standing
+# separate code; it doubles as the witness that this render really is flat.
+assert_has    `pwd`/test_build_gauss_flat/CIP.sub            "/bin/true"
+assert_has    `pwd`/test_build_gauss_flat/CIP_0.sub          "/bin/true"
+assert_absent `pwd`/test_build_gauss_flat/CIP_0.sub          "GaussianResampling"
+assert_absent `pwd`/test_build_gauss_flat/CIP_0.sub          "GenericCoordinates"
+assert_has    `pwd`/test_build_gauss_flat/CIP_worker0.sub    "GaussianResampling"
+assert_has    `pwd`/test_build_gauss_flat/CIP_1.sub          "/bin/true"
+assert_has    `pwd`/test_build_gauss_flat/CIP_worker1.sub    "GenericCoordinates"
+echo "OK: flat explode mode keeps the no-op /bin/true master for G and non-G groups"
+
 echo "test-build.sh: all pipeline-build checks passed"

.travis/test-calmarg-gpu.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Calibration-marginalization CUDA gate: exercises the fused GPU kernels
+# (cuda_Q_fused_calmarg[_distmarg].cu) via the fused reduction against the
+# brute-force reference, including the per-realization self-term.  Requires a GPU
+# + cupy; run on the GitLab gpu runner.  Any nonzero exit fails the job.
+set -euo pipefail
+PY="${PYTHON:-python}"
+command -v "$PY" >/dev/null 2>&1 || PY="$(command -v python3)"
+CODE="MonteCarloMarginalizeCode/Code"
+export OMP_NUM_THREADS=1
+( cd "$CODE" \
+  && "$PY" -m RIFT.calmarg.test_selfterm_reduction --backend gpu \
+  && "$PY" -m RIFT.calmarg.backtest_calmarg --backend gpu --n-cal 12 --methods reference,in_loop_B,in_loop_C \
+  && "$PY" -m RIFT.calmarg.backtest_calmarg --backend gpu --n-cal 12 --loglikelihood distmarg --methods reference,in_loop_B,in_loop_C )
+echo "calmarg GPU (CUDA) regression gate: PASS"

.travis/test-calmarg.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Calibration-marginalization regression gate (CPU).  Covers the in-loop calmarg
+# reduction and the per-realization self-term fix: precompute time-alignment +
+# identity-cal cross terms, the loop/fused reduction vs a brute-force reference
+# (incl. n_cal==1), the low-rank SVD self-term basis vs a direct band integral, and
+# the backtest of the cal reduction (default + distance-marginalization helpers).
+# Any nonzero exit fails the job (set -e).  GPU/CUDA paths are exercised separately
+# on hardware; here every check runs on the numpy backend.
+set -euo pipefail
+
+PY="${PYTHON:-python}"
+command -v "$PY" >/dev/null 2>&1 || PY="$(command -v python3)"
+CODE="MonteCarloMarginalizeCode/Code"
+export OMP_NUM_THREADS=1
+
+# precompute alignment + identity-cal self-term cross terms == baseline
+"$PY" "$CODE/RIFT/calmarg/test_precompute_alignment.py"
+
+# reduction + self-term basis + backtest run as modules from the code root
+( cd "$CODE" \
+  && "$PY" -m RIFT.calmarg.test_selfterm_basis \
+  && "$PY" -m RIFT.calmarg.test_selfterm_reduction --backend cpu \
+  && "$PY" -m RIFT.calmarg.test_calmarg_reduction \
+  && "$PY" -m RIFT.calmarg.backtest_calmarg --backend cpu --n-cal 8 --methods reference,in_loop_B \
+  && "$PY" -m RIFT.calmarg.backtest_calmarg --backend cpu --n-cal 8 --loglikelihood distmarg --methods reference,in_loop_B )
+
+echo "calmarg CPU regression gate: PASS"

.travis/test-integrate.sh

@@ -32,6 +32,12 @@ print(f"GPU preflight OK: cupy={cupy.__version__}, cuda_devices={n_devices}")
 PY
 fi
 
+# Unit/regression tests for the sampler helpers themselves (fast, no data needed).
+# The extrinsic "zoom box" limits under the cosine samplers live here: they are pure
+# coordinate-transform + prior-mass identities, so they belong with the integrator gate
+# rather than with the end-to-end run tests.
+python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_limit_cosine_samplers.py
+
 python MonteCarloMarginalizeCode/Code/test/test_mcsamplerEnsemble_extended.py --as-test --n-max 100000
 
 python MonteCarloMarginalizeCode/Code/test/test_mcsamplerEnsemble_extended.py --as-test --n-max 100000 --use-lnL