From 561483d1ab6d966cb7b54f339ad2d0475db16fde Mon Sep 17 00:00:00 2001 From: Felipe Bonchristiano Date: Wed, 26 Aug 2026 11:41:48 -0500 Subject: [PATCH 1/4] Fix ece_confidence_binary crash on binary tasks The function indexed prob[:,0]/label[:,0], requiring 2D arrays, but its only caller (binary_metrics_fn) passes 1D positive-class probs and 1D 0/1 labels, so ECE/ECE_adapt always raised IndexError. Use the positive class (class 1) as confidence and the 0/1 label as target, tolerating 1D and 2D inputs. Also corrects the class-0 vs class-1 indexing. --- pyhealth/metrics/calibration.py | 11 +++++++- tests/core/test_calibration_binary_ece.py | 33 +++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/core/test_calibration_binary_ece.py diff --git a/pyhealth/metrics/calibration.py b/pyhealth/metrics/calibration.py index 32e27b617..a77cf2062 100644 --- a/pyhealth/metrics/calibration.py +++ b/pyhealth/metrics/calibration.py @@ -147,7 +147,16 @@ def ece_confidence_binary(prob:np.ndarray, label:np.ndarray, bins=20, adaptive=F of points. Defaults to False. """ - df = pd.DataFrame({'acc': label[:,0], 'conf': prob[:,0]}) + prob = np.asarray(prob) + label = np.asarray(label) + # Confidence in the positive class (class 1). Accept either a 1D array of + # positive-class probabilities (as binary_metrics_fn passes) or a 2D + # (N, C) probability matrix. + conf = prob[:, 1] if prob.ndim > 1 else prob + # Whether each sample truly belongs to the positive class. Accept either a + # 1D 0/1 label array or a 2D one-hot label matrix. + acc = label[:, 1] if label.ndim > 1 else label + df = pd.DataFrame({'acc': acc, 'conf': conf}) return _ECE_confidence(df, bins, adaptive)[1] def ece_classwise(prob, label, bins=20, threshold=0., adaptive=False): diff --git a/tests/core/test_calibration_binary_ece.py b/tests/core/test_calibration_binary_ece.py new file mode 100644 index 000000000..e22a7920e --- /dev/null +++ b/tests/core/test_calibration_binary_ece.py @@ -0,0 +1,33 @@ +import unittest +from unittest.mock import patch + +import numpy as np + +from pyhealth.metrics import binary_metrics_fn +from pyhealth.metrics.calibration import ece_confidence_binary + + +class TestBinaryECE(unittest.TestCase): + def test_binary_metrics_fn_ece_does_not_crash(self): + y_true = np.array([0, 0, 1, 1, 0, 1]) + y_prob = np.array([0.1, 0.4, 0.35, 0.8, 0.2, 0.7]) + for metric in ("ECE", "ECE_adapt"): + out = binary_metrics_fn(y_true, y_prob, metrics=[metric]) + self.assertIn(metric, out) + self.assertTrue(np.isfinite(out[metric])) + self.assertGreaterEqual(out[metric], 0.0) + self.assertLessEqual(out[metric], 1.0) + + def test_two_dim_inputs_use_positive_class(self): + prob = np.array([[0.2, 0.8], [0.7, 0.3]]) + label = np.array([[0, 1], [1, 0]]) + + with patch( + "pyhealth.metrics.calibration._ECE_confidence", + return_value=(None, 0.0), + ) as ece: + ece_confidence_binary(prob, label) + + frame = ece.call_args.args[0] + np.testing.assert_array_equal(frame["conf"].to_numpy(), prob[:, 1]) + np.testing.assert_array_equal(frame["acc"].to_numpy(), label[:, 1]) From 0af4d2b843cd72273dcde0edd13d4ffb08819aa6 Mon Sep 17 00:00:00 2001 From: Felipe Bonchristiano Date: Thu, 27 Aug 2026 17:27:41 -0500 Subject: [PATCH 2/4] update new test confidence_ece test file --- pyhealth/metrics/calibration.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pyhealth/metrics/calibration.py b/pyhealth/metrics/calibration.py index a77cf2062..13c53dc1a 100644 --- a/pyhealth/metrics/calibration.py +++ b/pyhealth/metrics/calibration.py @@ -149,13 +149,10 @@ def ece_confidence_binary(prob:np.ndarray, label:np.ndarray, bins=20, adaptive=F prob = np.asarray(prob) label = np.asarray(label) - # Confidence in the positive class (class 1). Accept either a 1D array of - # positive-class probabilities (as binary_metrics_fn passes) or a 2D - # (N, C) probability matrix. + conf = prob[:, 1] if prob.ndim > 1 else prob - # Whether each sample truly belongs to the positive class. Accept either a - # 1D 0/1 label array or a 2D one-hot label matrix. acc = label[:, 1] if label.ndim > 1 else label + df = pd.DataFrame({'acc': acc, 'conf': conf}) return _ECE_confidence(df, bins, adaptive)[1] From 5f63ae770c2cef40a013e3211dfc7c9702c73417 Mon Sep 17 00:00:00 2001 From: Felipe Bonchristiano Date: Thu, 27 Aug 2026 17:33:04 -0500 Subject: [PATCH 3/4] Document binary ECE usage --- pyhealth/metrics/calibration.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pyhealth/metrics/calibration.py b/pyhealth/metrics/calibration.py index 13c53dc1a..107079f5c 100644 --- a/pyhealth/metrics/calibration.py +++ b/pyhealth/metrics/calibration.py @@ -137,6 +137,12 @@ def ece_confidence_binary(prob:np.ndarray, label:np.ndarray, bins=20, adaptive=F Similar to :func:`ece_confidence_multiclass`, but on class 1 instead of the top-prediction. + Examples: + >>> prob = np.array([0.1, 0.8]) + >>> label = np.array([0, 1]) + >>> ece = ece_confidence_binary(prob, label, bins=2) + >>> 0.0 <= ece <= 1.0 + True Args: prob (np.ndarray): (N, C) From 15bcf4add33ca63dec63dadb203eee42456518c2 Mon Sep 17 00:00:00 2001 From: Felipe Bonchristiano Date: Fri, 28 Aug 2026 17:06:44 -0500 Subject: [PATCH 4/4] Handle single-column binary ECE inputs --- pyhealth/metrics/calibration.py | 4 ++-- tests/core/test_calibration_binary_ece.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pyhealth/metrics/calibration.py b/pyhealth/metrics/calibration.py index 107079f5c..99f241e32 100644 --- a/pyhealth/metrics/calibration.py +++ b/pyhealth/metrics/calibration.py @@ -156,8 +156,8 @@ def ece_confidence_binary(prob:np.ndarray, label:np.ndarray, bins=20, adaptive=F prob = np.asarray(prob) label = np.asarray(label) - conf = prob[:, 1] if prob.ndim > 1 else prob - acc = label[:, 1] if label.ndim > 1 else label + conf = prob[:, 0 if prob.shape[1] == 1 else 1] if prob.ndim > 1 else prob + acc = label[:, 0 if label.shape[1] == 1 else 1] if label.ndim > 1 else label df = pd.DataFrame({'acc': acc, 'conf': conf}) return _ECE_confidence(df, bins, adaptive)[1] diff --git a/tests/core/test_calibration_binary_ece.py b/tests/core/test_calibration_binary_ece.py index e22a7920e..0b28f02b5 100644 --- a/tests/core/test_calibration_binary_ece.py +++ b/tests/core/test_calibration_binary_ece.py @@ -31,3 +31,17 @@ def test_two_dim_inputs_use_positive_class(self): frame = ece.call_args.args[0] np.testing.assert_array_equal(frame["conf"].to_numpy(), prob[:, 1]) np.testing.assert_array_equal(frame["acc"].to_numpy(), label[:, 1]) + + def test_single_column_inputs_use_only_column(self): + prob = np.array([[0.2], [0.7]]) + label = np.array([[0], [1]]) + + with patch( + "pyhealth.metrics.calibration._ECE_confidence", + return_value=(None, 0.0), + ) as ece: + ece_confidence_binary(prob, label) + + frame = ece.call_args.args[0] + np.testing.assert_array_equal(frame["conf"].to_numpy(), prob[:, 0]) + np.testing.assert_array_equal(frame["acc"].to_numpy(), label[:, 0])