From cc82990c8028bcfbe055e6f258c0963ada4c8f98 Mon Sep 17 00:00:00 2001 From: DucAnhValentinoNguyen Date: Sat, 12 Sep 2026 16:29:29 +0200 Subject: [PATCH 1/5] Include per-class rank in EmbeddingCollapseMetric aggregate score (Fixes #9116) Signed-off-by: DucAnhValentinoNguyen --- monai/metrics/embedding_collapse.py | 18 ++++++-- tests/metrics/test_embedding_collapse.py | 55 ++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/monai/metrics/embedding_collapse.py b/monai/metrics/embedding_collapse.py index b427660b90e..dc2654fe3b1 100644 --- a/monai/metrics/embedding_collapse.py +++ b/monai/metrics/embedding_collapse.py @@ -51,8 +51,10 @@ class EmbeddingCollapseMetric(Metric): Args: reduction: how to aggregate individual scores into ``aggregate``. ``"max"`` returns the worst-case score (recommended for - safety-critical use). ``"mean"`` returns the average of - available scores. ``"none"`` omits the ``aggregate`` key. + safety-critical use), including the most collapsed class. + ``"mean"`` returns the average of available scores, where all + per-class scores contribute a single term. ``"none"`` omits the + ``aggregate`` key. include_indicators: optional list of indicator names to compute. If ``None``, all applicable indicators are computed. Valid names: ``"centroid_similarity"``, ``"effective_rank"``, @@ -162,7 +164,8 @@ class centroids. ``None`` if fewer than 2 classes. ``None`` if ``target_embeddings`` not provided. - ``separation``: silhouette-based inter-class separation score. ``None`` if sklearn unavailable or fewer than 2 classes. - - ``aggregate``: reduced score. Omitted when ``reduction="none"``. + - ``aggregate``: reduced over the global indicators plus the worst + ``per_class_rank_``. Omitted when ``reduction="none"``. Raises: ValueError: if inputs are invalid (shape, reduction, indicators). @@ -180,6 +183,7 @@ class centroids. ``None`` if fewer than 2 classes. inc = set(include_indicators) if include_indicators is not None else None scores: dict[str, torch.Tensor | None] = {} + per_class_scores: dict[str, torch.Tensor | None] = {} # -- Label-dependent indicators if labels is not None: @@ -191,7 +195,8 @@ class centroids. ``None`` if fewer than 2 classes. scores["centroid_similarity"] = _centroid_similarity(emb, lbl) if inc is None or "per_class_rank" in inc: - scores.update(_per_class_rank(emb, lbl)) + per_class_scores = _per_class_rank(emb, lbl) + scores.update(per_class_scores) if inc is None or "separation" in inc: scores["separation"] = _separation(emb, lbl) @@ -218,6 +223,11 @@ class centroids. ``None`` if fewer than 2 classes. if reduction != "none": primary = {"centroid_similarity", "effective_rank_score", "domain_shift", "separation"} available = [v.to(device=emb.device) for k, v in scores.items() if k in primary and v is not None] + # Per-class collapse is reduced to its worst class before joining the pool, so that + # `mean` keeps one vote per indicator regardless of how many classes are present. + by_class = [v.to(device=emb.device) for v in per_class_scores.values() if v is not None] + if by_class: + available.append(torch.stack(by_class).max()) if not available: scores["aggregate"] = None elif reduction == "max": diff --git a/tests/metrics/test_embedding_collapse.py b/tests/metrics/test_embedding_collapse.py index b406f922157..36c0a7e36ae 100644 --- a/tests/metrics/test_embedding_collapse.py +++ b/tests/metrics/test_embedding_collapse.py @@ -441,5 +441,60 @@ def test_raises_without_sklearn(self): ) + +class TestAggregateIncludesPerClassRank(unittest.TestCase): + """Asymmetric collapse must reach the summary score, not only the per-class keys.""" + + @staticmethod + def _majority_healthy_minority_collapsed(n_major=200, n_minor=8, d=64, seed=17): + """Healthy majority class; minority class collapsed to a single point.""" + torch.manual_seed(seed) + major = torch.randn(n_major, d) + major[:, 0] -= 4.0 # centroid ~ -e1 + minor = torch.zeros(n_minor, d) + minor[:, 0] = 8.0 # identical rows, centroid = +e1 + emb = torch.cat([major, minor]) + lbl = torch.tensor([0] * n_major + [1] * n_minor, dtype=torch.long) + return emb, lbl + + def test_collapsed_minority_class_reaches_aggregate(self): + emb, lbl = self._majority_healthy_minority_collapsed() + scores = compute_embedding_collapse(emb, lbl, reduction="max") + self.assertAlmostEqual(float(scores["per_class_rank_1"]), 1.0, places=5) + self.assertAlmostEqual(float(scores["aggregate"]), 1.0, places=5) + + def test_max_aggregate_at_least_worst_per_class(self): + emb, lbl = self._majority_healthy_minority_collapsed() + scores = compute_embedding_collapse(emb, lbl, reduction="max") + per_class = [ + float(v) for k, v in scores.items() if k.startswith("per_class_rank_") and v is not None + ] + self.assertGreaterEqual(float(scores["aggregate"]), max(per_class)) + + def test_global_indicators_miss_the_collapse(self): + """Guard: the fixture is only meaningful while the global view stays healthy.""" + emb, lbl = self._majority_healthy_minority_collapsed() + scores = compute_embedding_collapse(emb, lbl, reduction="none") + self.assertLess(float(scores["effective_rank_score"]), 0.2) + self.assertLess(float(scores["centroid_similarity"]), 0.2) + + def test_per_class_contributes_one_term_to_mean(self): + """Three classes must not outvote the global indicators 3:4 under `mean`.""" + torch.manual_seed(23) + emb = torch.cat([torch.randn(40, 32) + k * 6.0 for k in range(3)]) + lbl = torch.tensor([0] * 40 + [1] * 40 + [2] * 40, dtype=torch.long) + scores = compute_embedding_collapse(emb, lbl, reduction="mean") + globals_ = [ + float(scores[k]) + for k in ("centroid_similarity", "effective_rank_score", "separation") + if scores.get(k) is not None + ] + per_class = [ + float(v) for k, v in scores.items() if k.startswith("per_class_rank_") and v is not None + ] + expected = (sum(globals_) + max(per_class)) / (len(globals_) + 1) + self.assertAlmostEqual(float(scores["aggregate"]), expected, places=5) + + if __name__ == "__main__": unittest.main() From f258f3e53da99cdce55c25daef32b910b9ee9509 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:33:43 +0000 Subject: [PATCH 2/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/metrics/test_embedding_collapse.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/metrics/test_embedding_collapse.py b/tests/metrics/test_embedding_collapse.py index 36c0a7e36ae..47ef8261952 100644 --- a/tests/metrics/test_embedding_collapse.py +++ b/tests/metrics/test_embedding_collapse.py @@ -441,7 +441,6 @@ def test_raises_without_sklearn(self): ) - class TestAggregateIncludesPerClassRank(unittest.TestCase): """Asymmetric collapse must reach the summary score, not only the per-class keys.""" @@ -466,9 +465,7 @@ def test_collapsed_minority_class_reaches_aggregate(self): def test_max_aggregate_at_least_worst_per_class(self): emb, lbl = self._majority_healthy_minority_collapsed() scores = compute_embedding_collapse(emb, lbl, reduction="max") - per_class = [ - float(v) for k, v in scores.items() if k.startswith("per_class_rank_") and v is not None - ] + per_class = [float(v) for k, v in scores.items() if k.startswith("per_class_rank_") and v is not None] self.assertGreaterEqual(float(scores["aggregate"]), max(per_class)) def test_global_indicators_miss_the_collapse(self): @@ -489,9 +486,7 @@ def test_per_class_contributes_one_term_to_mean(self): for k in ("centroid_similarity", "effective_rank_score", "separation") if scores.get(k) is not None ] - per_class = [ - float(v) for k, v in scores.items() if k.startswith("per_class_rank_") and v is not None - ] + per_class = [float(v) for k, v in scores.items() if k.startswith("per_class_rank_") and v is not None] expected = (sum(globals_) + max(per_class)) / (len(globals_) + 1) self.assertAlmostEqual(float(scores["aggregate"]), expected, places=5) From 371afb61de325a260debe7c21437ae9c0d19b5ed Mon Sep 17 00:00:00 2001 From: Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> Date: Wed, 23 Sep 2026 12:23:00 +0200 Subject: [PATCH 3/5] Address CodeRabbit review: add docstrings to new test helper/methods Adds Google-style Args/Returns to _majority_healthy_minority_collapsed and one-line docstrings to test_collapsed_minority_class_reaches_aggregate and test_max_aggregate_at_least_worst_per_class, per CodeRabbit's review comment. No behavioural changes. Signed-off-by: Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> --- tests/metrics/test_embedding_collapse.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/metrics/test_embedding_collapse.py b/tests/metrics/test_embedding_collapse.py index 47ef8261952..34df07bc8e9 100644 --- a/tests/metrics/test_embedding_collapse.py +++ b/tests/metrics/test_embedding_collapse.py @@ -446,7 +446,19 @@ class TestAggregateIncludesPerClassRank(unittest.TestCase): @staticmethod def _majority_healthy_minority_collapsed(n_major=200, n_minor=8, d=64, seed=17): - """Healthy majority class; minority class collapsed to a single point.""" + """Build a two-class embedding set: a healthy majority class and a minority + class collapsed to a single point. + + Args: + n_major: number of samples in the healthy majority class. + n_minor: number of samples in the collapsed minority class. + d: embedding dimensionality. + seed: seed for the majority class's random draw. + + Returns: + Tuple of ``(embeddings, labels)`` with shapes ``[n_major + n_minor, d]`` + and ``[n_major + n_minor]``. + """ torch.manual_seed(seed) major = torch.randn(n_major, d) major[:, 0] -= 4.0 # centroid ~ -e1 @@ -457,12 +469,14 @@ def _majority_healthy_minority_collapsed(n_major=200, n_minor=8, d=64, seed=17): return emb, lbl def test_collapsed_minority_class_reaches_aggregate(self): + """A fully collapsed minority class should drive `aggregate` to 1.0 under `max`.""" emb, lbl = self._majority_healthy_minority_collapsed() scores = compute_embedding_collapse(emb, lbl, reduction="max") self.assertAlmostEqual(float(scores["per_class_rank_1"]), 1.0, places=5) self.assertAlmostEqual(float(scores["aggregate"]), 1.0, places=5) def test_max_aggregate_at_least_worst_per_class(self): + """`aggregate` under `max` must never fall below the worst per-class score.""" emb, lbl = self._majority_healthy_minority_collapsed() scores = compute_embedding_collapse(emb, lbl, reduction="max") per_class = [float(v) for k, v in scores.items() if k.startswith("per_class_rank_") and v is not None] From d47b32d482570b60cf3f748c0db7c5a34953adfb Mon Sep 17 00:00:00 2001 From: Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:01:05 +0200 Subject: [PATCH 4/5] Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> DCO Remediation Commit for Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> I, Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com>, hereby add my Signed-off-by to this commit: 371afb61de325a260debe7c21437ae9c0d19b5e Signed-off-by: Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> Signed-off-by: Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> --- tests/metrics/test_embedding_collapse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/metrics/test_embedding_collapse.py b/tests/metrics/test_embedding_collapse.py index 34df07bc8e9..f168499a093 100644 --- a/tests/metrics/test_embedding_collapse.py +++ b/tests/metrics/test_embedding_collapse.py @@ -469,14 +469,14 @@ class collapsed to a single point. return emb, lbl def test_collapsed_minority_class_reaches_aggregate(self): - """A fully collapsed minority class should drive `aggregate` to 1.0 under `max`.""" + """A fully collapsed minority class should drive ``aggregate`` to 1.0 under ``max``.""" emb, lbl = self._majority_healthy_minority_collapsed() scores = compute_embedding_collapse(emb, lbl, reduction="max") self.assertAlmostEqual(float(scores["per_class_rank_1"]), 1.0, places=5) self.assertAlmostEqual(float(scores["aggregate"]), 1.0, places=5) def test_max_aggregate_at_least_worst_per_class(self): - """`aggregate` under `max` must never fall below the worst per-class score.""" + """``aggregate`` under ``max`` must never fall below the worst per-class score.""" emb, lbl = self._majority_healthy_minority_collapsed() scores = compute_embedding_collapse(emb, lbl, reduction="max") per_class = [float(v) for k, v in scores.items() if k.startswith("per_class_rank_") and v is not None] From 9d06f81df59fcb868fb71312d533343fa1f3e633 Mon Sep 17 00:00:00 2001 From: Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:03:13 +0200 Subject: [PATCH 5/5] Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> DCO Remediation Commit for Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> I, Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com>, hereby add my Signed-off-by to this commit: 371afb61de325a260debe7c21437ae9c0d19b5ed Signed-off-by: Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> Signed-off-by: Duc-Anh Valentino Nguyen <61320780+DucAnhValentinoNguyen@users.noreply.github.com> --- tests/metrics/test_embedding_collapse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/metrics/test_embedding_collapse.py b/tests/metrics/test_embedding_collapse.py index f168499a093..653e4384f83 100644 --- a/tests/metrics/test_embedding_collapse.py +++ b/tests/metrics/test_embedding_collapse.py @@ -457,7 +457,7 @@ class collapsed to a single point. Returns: Tuple of ``(embeddings, labels)`` with shapes ``[n_major + n_minor, d]`` - and ``[n_major + n_minor]``. + and ``[n_major + n_minor]``, where ``labels`` has dtype ``torch.long``. """ torch.manual_seed(seed) major = torch.randn(n_major, d)