Pool replicate samples instead of refusing the run

.changeset/pool-replicate-samples.md

@@ -0,0 +1,27 @@
+---
+'@platforma-open/milaboratories.sort-seq-analysis.software': patch
+'@platforma-open/milaboratories.sort-seq-analysis.model': patch
+'@platforma-open/milaboratories.sort-seq-analysis.ui': patch
+'@platforma-open/milaboratories.sort-seq-analysis.block': patch
+---
+
+Pool replicate samples instead of refusing the run
+
+A condition-and-gate group holding more than one sample used to fail the run. Its reads are
+now summed, matching what `titeseq-analysis` and `clonotype-enrichment` already do.
+
+The pooling is reported rather than silent, which is the objection the original refusal was
+raised against: the manifest gains a `pooledGroups` list, the run log names every merged
+group and its samples, and the Run statistics dialog shows a warning listing them. The dialog
+resolves sample ids to sample labels through the sample axis's label column, so the alert
+names samples the way the user does. Only retained conditions are reported — an excluded
+condition is not part of the run. Where replicates supplied different sort fractions, the
+non-null values are averaged and that is flagged separately.
+
+Two things broke quietly on replicate samples and are fixed at the source, by pooling before
+anything reads the table: the read-distribution file emitted duplicate `(variantKey, gate)`
+keys, and the sort-fraction check counted a twice-collected gate's fraction twice and refused
+runs that were entirely valid.
+
+This is temporary. It does not decide whether replicates should be pooled at all rather than
+scored separately, nor what to do when they disagree on the sort fraction.

model/src/columns.ts

@@ -31,6 +31,8 @@ export const PColumnName = {
   MutationCount: "pl7.app/repertoire/mutationCount",
   /** The variant axis's label column — shown as "Variant Id". */
   VariantLabel: "pl7.app/label",
+  /** The **sample** axis's label column. Same name as `VariantLabel`; the axis picks one. */
+  SampleLabel: "pl7.app/label",
   /** The per-variant mutation list, shown as "Mutations". */
   Mutations: "pl7.app/repertoire/mutations",
 } as const;
@@ -104,6 +106,16 @@ export const metadataSelector: AnchoredPColumnSelector = {
   name: PColumnName.Metadata,
 };
 
+/**
+ * The sample axis's label column. `pl7.app/isLabel` is required alongside the name, which the
+ * variant label column shares.
+ */
+export const sampleLabelSelector: AnchoredPColumnSelector = {
+  axes: [{ anchor: "main", idx: 0 }],
+  name: PColumnName.SampleLabel,
+  annotations: { "pl7.app/isLabel": "true" },
+};
+
 /**
  * The variants' mutation count. Resolved in the workflow rather than here — this constant
  * exists so the model and the workflow state the same predicate, and so a reader can see

model/src/index.ts

@@ -12,7 +12,14 @@ import {
   type SUniversalPColumnId,
 } from "@platforma-sdk/model";
 import { kind } from "@platforma-open/milaboratories.sort-seq-analysis.kind";
-import { Annotation, FacsBin, isAbundanceAnchor, metadataSelector, PColumnName } from "./columns";
+import {
+  Annotation,
+  FacsBin,
+  isAbundanceAnchor,
+  metadataSelector,
+  PColumnName,
+  sampleLabelSelector,
+} from "./columns";
 import { blockDataModel } from "./dataModel";
 import type { BlockArgs, BlockData, RunManifest } from "./types";
 
@@ -24,10 +31,9 @@ export * from "./types";
  * Every configuration rule, checked here and **nowhere else**. These six are decidable from the
  * arguments and snapshotted column values alone, so they are refused before the run starts.
  *
- * The two data-value rules — sort fractions, one sample per condition-and-gate group — belong
- * to the computation and are deliberately not approximated here: a duplicated rule is one that
- * will disagree, and it fails by drifting looser, so the settings pass and the run fails anyway
- * with a different message.
+ * The one data-value rule — sort fractions — belongs to the computation and is deliberately not
+ * approximated here: a duplicated rule is one that will disagree, and it fails by drifting
+ * looser, so the settings pass and the run fails anyway with a different message.
  */
 export function settingsIssues(data: BlockData): string[] {
   const issues: string[] = [];
@@ -233,6 +239,29 @@ export const platforma = BlockModelV3.create({ dataModel: blockDataModel, kind }
     return ctx.createPFrame(columns as PColumn<PColumnValues>[]);
   })
 
+  /**
+   * The sample label column, so the UI can resolve the pooling report's `PlId`s to sample names.
+   * Kept out of `metadataColumnsPframe`, whose columns are the roles the pickers offer.
+   */
+  .output("sampleLabelPframe", (ctx) => {
+    const anchor = ctx.data.abundanceRef;
+    if (!anchor) return undefined;
+
+    const columns = ctx.resultPool.getAnchoredPColumns({ main: anchor }, [sampleLabelSelector]);
+    if (!columns || columns.length === 0) return undefined;
+
+    return ctx.createPFrame(columns as PColumn<PColumnValues>[]);
+  })
+
+  /** The `PObjectId` for reading that column out of the frame above. */
+  .output("sampleLabelColumnId", (ctx) => {
+    const anchor = ctx.data.abundanceRef;
+    if (!anchor) return undefined;
+
+    const columns = ctx.resultPool.getAnchoredPColumns({ main: anchor }, [sampleLabelSelector]);
+    return columns?.[0]?.id;
+  })
+
   // ---------------------------------------------------------------------------
   // The three views.
   // ---------------------------------------------------------------------------

model/src/types.ts

@@ -105,6 +105,19 @@ export type GateCollected = {
   depth: number;
 };
 
+/** One condition-and-gate group whose reads were pooled across several samples. */
+export type PooledGroup = {
+  condition: string;
+  gate: string;
+  /** Sample **ids**, sorted. The UI resolves them to labels via `sampleLabelPframe`. */
+  samples: string[];
+  /**
+   * The replicates supplied different fractions and the non-null values were averaged. Absent
+   * in the uncorrected mode.
+   */
+  sortFractionsDiffer?: boolean;
+};
+
 /** The manifest's per-condition entry. */
 export type ConditionSummary = {
   /** Verbatim, exactly as it appears in the metadata column. */
@@ -141,5 +154,7 @@ export type RunManifest = {
    * no mutation-count table at all, which is a third state rather than a reason.
    */
   parentAbsenceReason: string | null;
+  /** Retained conditions only; empty on a run with no replicates. */
+  pooledGroups: PooledGroup[];
   conditions: ConditionSummary[];
 };

software/src/errors.py

@@ -6,10 +6,9 @@
 condition excluded, a negative floor — are refused by the block model before the run
 starts and never arrive here.
 
-What arrives here is the data-value class: a sort-fraction set breaking
-`sort-fraction-values`, and more than one sample in a condition-and-gate group. Those
-raise `Refusal`, which `main` turns into a non-zero exit that names the offending
-values and writes no file.
+What arrives here is the data-value class, which is a single rule: a sort-fraction set
+breaking `sort-fraction-values`. It raises `Refusal`, which `main` turns into a non-zero
+exit that names the offending values and writes no file.
 
 A `Refusal` is therefore *not* an internal assertion. Anything raised as some other
 exception type is a bug in this package or a caller violating the interface, and is

software/src/main.py

@@ -90,6 +90,16 @@ def _report(manifest: dict) -> None:
         detail = reason if reason is not None else "no mutation-count table supplied"
         print(f"Parent row: not identified ({detail})")
 
+    for entry in manifest["pooledGroups"]:
+        samples = " + ".join(entry["samples"])
+        line = (
+            f"Pooled condition {entry['condition']!r} gate {entry['gate']!r}: "
+            f"{len(entry['samples'])} samples merged ({samples})"
+        )
+        if entry.get("sortFractionsDiffer"):
+            line += " — WARNING: replicates supplied different sort fractions; averaged"
+        print(line)
+
     for entry in manifest["conditions"]:
         gates = ", ".join(f"{gate['gate']}={gate['depth']}" for gate in entry["gatesCollected"])
         summary = (

software/src/pipeline.py

@@ -32,18 +32,22 @@
 )
 from io_layer import distribution_file_name, score_file_name, write_manifest, write_table
 from params import Params
-from validate import check_one_sample_per_group, check_sort_fractions
+from pooling import pool_replicates
+from validate import check_sort_fractions
 
 
 def run(reads: pl.DataFrame, variants: pl.DataFrame | None, params: Params, out_dir: Path) -> dict:
     """Score every retained condition and write every file. Returns the manifest."""
     in_scope = selected_gates(reads, params)
+
+    # Before pooling: the report is confined to retained conditions.
     retained = retained_conditions(in_scope, params)
 
-    # Both refusals run over the whole run before anything is written, so a failure leaves
-    # nothing partial behind. Over the in-scope rows only: a second sample in a gate the run
-    # does not cover, or a missing sort fraction on one, is not this run's problem.
-    check_one_sample_per_group(in_scope, retained)
+    # Must run before anything else reads the table — see `pooling`. Over the in-scope rows
+    # only, so a second sample in a gate the run does not cover is not pooled.
+    in_scope, pooled_groups = pool_replicates(in_scope, params.sort_fraction_column, retained)
+
+    # Over the whole run before anything is written, so a failure leaves nothing partial.
     if params.sort_fraction_column is not None:
         check_sort_fractions(in_scope, params.sort_fraction_column, retained)
 
@@ -61,6 +65,8 @@ def run(reads: pl.DataFrame, variants: pl.DataFrame | None, params: Params, out_
         # Null where there was no mutation-count table at all: the two reasons below are
         # for a table that exists and does not identify a single parent.
         "parentAbsenceReason": parent.absence_reason,
+        # Retained conditions only; empty on a run with no replicates.
+        "pooledGroups": pooled_groups,
         "conditions": conditions,
     }
     write_manifest(manifest, out_dir)

software/src/pooling.py

@@ -0,0 +1,117 @@
+"""Pool replicate samples that share a condition-and-gate group.
+
+Pooling here, once, before anything else reads the table keeps the rest of the package on its
+one-row-per-(gate, variant) grain. Two places break silently otherwise:
+`scoring.read_distribution` emits duplicate (variantKey, gate) keys, and
+`check_sort_fractions` counts a twice-collected gate's fraction twice and over-sums.
+
+Temporary. Whether replicates should be pooled at all rather than scored separately is open,
+as is what to do when they disagree on the sort fraction (here: averaged and flagged).
+"""
+
+from __future__ import annotations
+
+import polars as pl
+from constants import (
+    COL_CONDITION,
+    COL_GATE,
+    COL_READS,
+    COL_SAMPLE,
+    COL_VARIANT,
+)
+
+# A single-sample group keeps its bare sample id, so pooling is a no-op there.
+_LABEL_SEPARATOR = "+"
+
+_SAMPLES = "_samples"
+_FRACTION_VALUES = "_fractionValues"
+
+
+def pool_replicates(
+    reads: pl.DataFrame, sort_fraction_column: str | None, retained_conditions: list[str]
+) -> tuple[pl.DataFrame, list[dict]]:
+    """Collapse to one row per (condition, gate, variant): reads summed, sort fraction the mean
+    of the group's non-null values, other metadata the first in sample-id order, sample ids
+    joined. Returns the pooled table and the groups that held more than one sample.
+
+    Metadata resolves per (condition, gate), not per variant: a variant missing from one
+    replicate would give its gate a shorter label, and `check_sort_fractions` groups by sample
+    and would then see that gate twice.
+
+    A `sort_fraction_column` the table does not carry is left for `check_sort_fractions` to
+    refuse, rather than pre-empted here with a polars traceback.
+
+    Only retained conditions are reported. Excluded rows are still pooled; nothing reads them.
+    """
+    fraction = sort_fraction_column if sort_fraction_column in reads.columns else None
+
+    # Read off the frame rather than listed: the workflow decides which metadata columns the
+    # reads table carries, and one named here but not there would vanish from the run.
+    fixed = {COL_CONDITION, COL_GATE, COL_VARIANT, COL_SAMPLE, COL_READS}
+    passthrough = [name for name in reads.columns if name not in fixed and name != fraction]
+
+    # Deduplicated to one row per sample first: metadata repeats across the sample's variant
+    # rows, so aggregating the raw rows would weight a sample by how many variants it detected.
+    metadata_columns = [COL_CONDITION, COL_GATE, COL_SAMPLE, *passthrough]
+    if fraction is not None:
+        metadata_columns.append(fraction)
+
+    by_sample = pl.col(COL_SAMPLE).sort()
+    meta_aggs = [pl.col(COL_SAMPLE).unique().sort().alias(_SAMPLES)]
+    if fraction is not None:
+        meta_aggs.append(pl.col(fraction).alias(_FRACTION_VALUES))
+    # Sample-id order, so the pick is deterministic rather than row-order dependent.
+    meta_aggs += [pl.col(name).sort_by(by_sample).first().alias(name) for name in passthrough]
+
+    group_meta = (
+        reads.select(metadata_columns)
+        .unique()
+        .group_by(COL_CONDITION, COL_GATE)
+        .agg(*meta_aggs)
+        .sort(COL_CONDITION, COL_GATE)
+    )
+
+    report = _report(group_meta, fraction, retained_conditions)
+
+    resolved = [pl.col(_SAMPLES).list.join(_LABEL_SEPARATOR).alias(COL_SAMPLE), *passthrough]
+    if fraction is not None:
+        resolved.append(pl.col(_FRACTION_VALUES).list.drop_nulls().list.mean().alias(fraction))
+    resolved_meta = group_meta.select(COL_CONDITION, COL_GATE, *resolved)
+
+    pooled = (
+        reads.group_by(COL_CONDITION, COL_GATE, COL_VARIANT)
+        .agg(pl.col(COL_READS).sum().alias(COL_READS))
+        .join(resolved_meta, on=[COL_CONDITION, COL_GATE], how="left")
+        .select(reads.columns)
+        .sort(COL_CONDITION, COL_GATE, COL_VARIANT)
+    )
+    return pooled, report
+
+
+def _report(
+    group_meta: pl.DataFrame, sort_fraction_column: str | None, retained_conditions: list[str]
+) -> list[dict]:
+    """The replicated groups of retained conditions, in condition-and-gate order.
+
+    `sortFractionsDiffer` means the group's replicates supplied different fractions, which is
+    either two separate sorts or wrong metadata — the fraction is a property of the gate.
+    """
+    replicated = group_meta.filter(
+        (pl.col(_SAMPLES).list.len() > 1)
+        & pl.col(COL_CONDITION).is_in(retained_conditions)
+    )
+    if replicated.height == 0:
+        return []
+
+    entries = []
+    for row in replicated.iter_rows(named=True):
+        entry = {
+            "condition": row[COL_CONDITION],
+            "gate": row[COL_GATE],
+            "samples": list(row[_SAMPLES]),
+        }
+        if sort_fraction_column is not None:
+            supplied = [value for value in row[_FRACTION_VALUES] if value is not None]
+            entry["sortFractionsDiffer"] = len(set(supplied)) > 1
+        entries.append(entry)
+    return entries