From 074df338e03b4cc783699eaf071aac59bc75606b Mon Sep 17 00:00:00 2001 From: Zhenchao Ni Date: Fri, 31 Jul 2026 13:11:36 +0800 Subject: [PATCH] fix(datasets): use fully qualified HF dataset ids Bare canonical dataset ids are no longer resolvable by huggingface_hub, which now requires a 'namespace/name' repository id. Loading 'glue' raises HfUriError, so the calibration dataset silently fell back to RandomDataset for every text task. Use 'nyu-mll/glue' and 'cardiffnlp/tweet_eval' instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../text-classification_fp16_config.json | 2 +- .../text-classification_w8a16_config.json | 2 +- scripts/e2e_eval/cache/baseline_cache.json | 8 ++++---- scripts/e2e_eval/testsets/models_with_acc.json | 2 +- src/winml/modelkit/commands/eval.py | 2 +- src/winml/modelkit/datasets/text.py | 11 ++++++++--- src/winml/modelkit/eval/config.py | 2 +- .../integration/datasets/test_text_classification.py | 10 +++++----- tests/unit/eval/test_eval.py | 6 +++--- 9 files changed, 25 insertions(+), 20 deletions(-) diff --git a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json index 8de0a638d..683080239 100644 --- a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json +++ b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json @@ -53,7 +53,7 @@ "eval": { "task": "text-classification", "dataset": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "samples": 100, "columns_mapping": { diff --git a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json index eeb1fff26..cd55911db 100644 --- a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json +++ b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json @@ -70,7 +70,7 @@ "eval": { "task": "text-classification", "dataset": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "samples": 1000, "columns_mapping": { diff --git a/scripts/e2e_eval/cache/baseline_cache.json b/scripts/e2e_eval/cache/baseline_cache.json index 8acbbe305..7590060d4 100644 --- a/scripts/e2e_eval/cache/baseline_cache.json +++ b/scripts/e2e_eval/cache/baseline_cache.json @@ -9,7 +9,7 @@ "elapsed": 59.2, "command": "python.exe run_pytorch_baseline.py --model finbert --task text-classification --device cpu --num-samples 1000 --dataset finbert_dataset --split val" }, - "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|tweet_eval|sentiment||1000": { + "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|cardiffnlp/tweet_eval|sentiment||1000": { "status": "PASS", "metric": { "metric": "accuracy", @@ -17,7 +17,7 @@ "num_samples": 1000 }, "elapsed": 91.5, - "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 1000 --dataset tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"}" + "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 1000 --dataset cardiffnlp/tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"}" }, "distilbert/distilbert-base-uncased-finetuned-sst-2-english|text-classification|nyu-mll/glue|sst2||1000": { "status": "PASS", @@ -959,7 +959,7 @@ "elapsed": 52.0, "command": "python.exe run_pytorch_baseline.py --model finbert --task text-classification --device cpu --num-samples 100 --dataset finbert_dataset --split val --winml-metric-key accuracy" }, - "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|tweet_eval|sentiment||100": { + "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|cardiffnlp/tweet_eval|sentiment||100": { "status": "PASS", "metric": { "metric": "accuracy", @@ -967,7 +967,7 @@ "num_samples": 100 }, "elapsed": 54.9, - "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 100 --dataset tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"} --winml-metric-key accuracy" + "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 100 --dataset cardiffnlp/tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"} --winml-metric-key accuracy" }, "distilbert/distilbert-base-uncased-finetuned-sst-2-english|text-classification|nyu-mll/glue|sst2||100": { "status": "PASS", diff --git a/scripts/e2e_eval/testsets/models_with_acc.json b/scripts/e2e_eval/testsets/models_with_acc.json index e21423161..03a136206 100644 --- a/scripts/e2e_eval/testsets/models_with_acc.json +++ b/scripts/e2e_eval/testsets/models_with_acc.json @@ -18,7 +18,7 @@ "group": "Top200", "priority": "P1", "dataset_config": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "metric": "accuracy", "columns_mapping": { diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index 07f1af61c..68d0c3e1c 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -45,7 +45,7 @@ "dataset_path", type=str, default=None, - help="HF dataset path (e.g. 'imagenet-1k', 'glue'). " + help="HF dataset path (e.g. 'imagenet-1k', 'nyu-mll/glue'). " "If omitted, uses a default dataset for the task.", ) @click.option( diff --git a/src/winml/modelkit/datasets/text.py b/src/winml/modelkit/datasets/text.py index e44f115e2..f3f709d69 100644 --- a/src/winml/modelkit/datasets/text.py +++ b/src/winml/modelkit/datasets/text.py @@ -27,6 +27,11 @@ logger = logging.getLogger(__name__) +# HF requires fully qualified repository ids ("namespace/name"); the legacy +# canonical alias "glue" is no longer resolvable. +DEFAULT_TEXT_DATASET = "nyu-mll/glue" +DEFAULT_TEXT_DATASET_SUBSET = "mrpc" + class TextDataset(BaseTaskDataset): """Dataset for text tasks with universal tokenization. @@ -56,7 +61,7 @@ def __init__( Args: model_name: HuggingFace model identifier - dataset_name: Dataset name (default: glue) + dataset_name: Dataset name (default: nyu-mll/glue) max_samples: Maximum samples (None = use all) data_split: Dataset split (default: train) max_length: Sequence length (default: from io_config or 128) @@ -85,8 +90,8 @@ def __init__( def _get_default_dataset(self) -> None: """Set default dataset if none specified.""" if self._dataset_name is None: - self._dataset_name = "glue" - self._config["subset"] = self._config.get("subset", "mrpc") + self._dataset_name = DEFAULT_TEXT_DATASET + self._config["subset"] = self._config.get("subset", DEFAULT_TEXT_DATASET_SUBSET) self._data_split = self._data_split or "train" def _resolve_max_length(self) -> None: diff --git a/src/winml/modelkit/eval/config.py b/src/winml/modelkit/eval/config.py index 1717ccdb5..03944010a 100644 --- a/src/winml/modelkit/eval/config.py +++ b/src/winml/modelkit/eval/config.py @@ -20,7 +20,7 @@ class DatasetConfig: """Dataset configuration, aligned with HF load_dataset() API. Attributes: - path: HF dataset path (e.g., "imagenet-1k", "glue"). + path: HF dataset path (e.g., "imagenet-1k", "nyu-mll/glue"). name: Config name for multi-config datasets (e.g., "mrpc"). split: Dataset split. samples: Number of samples to evaluate. diff --git a/tests/integration/datasets/test_text_classification.py b/tests/integration/datasets/test_text_classification.py index 9559d4006..d1a291d63 100644 --- a/tests/integration/datasets/test_text_classification.py +++ b/tests/integration/datasets/test_text_classification.py @@ -44,7 +44,7 @@ def test_default_seq_len_constant(self): assert TextDataset.DEFAULT_SEQ_LEN == 128 def test_default_dataset_glue_mrpc(self): - """Test default dataset is glue/mrpc when none specified.""" + """Test default dataset is nyu-mll/glue with the mrpc subset.""" from winml.modelkit.datasets import TextDataset dataset = TextDataset( @@ -52,7 +52,7 @@ def test_default_dataset_glue_mrpc(self): max_samples=5, ) - assert dataset.dataset_name == "glue" + assert dataset.dataset_name == "nyu-mll/glue" assert dataset.data_split == "train" def test_explicit_dataset_name(self): @@ -61,13 +61,13 @@ def test_explicit_dataset_name(self): dataset = TextDataset( model_name="bert-base-uncased", - dataset_name="glue", + dataset_name="nyu-mll/glue", data_split="validation", max_samples=5, subset="sst2", ) - assert dataset.dataset_name == "glue" + assert dataset.dataset_name == "nyu-mll/glue" assert dataset.data_split == "validation" def test_max_samples_limits_dataset_size(self): @@ -274,7 +274,7 @@ def test_single_sentence_detection_sst2(self): # GLUE/SST2 is a single sentence task dataset = TextDataset( model_name="bert-base-uncased", - dataset_name="glue", + dataset_name="nyu-mll/glue", data_split="train", max_samples=5, subset="sst2", diff --git a/tests/unit/eval/test_eval.py b/tests/unit/eval/test_eval.py index 6c5bb469b..dbf93b72e 100644 --- a/tests/unit/eval/test_eval.py +++ b/tests/unit/eval/test_eval.py @@ -752,7 +752,7 @@ def _fake_compute(**kw): model_id="test/model", task="text-classification", dataset=DatasetConfig( - path="glue", + path="nyu-mll/glue", name="mrpc", columns_mapping={"input_column": "sentence1", "second_input_column": "sentence2"}, ), @@ -803,7 +803,7 @@ def test_sets_padding_for_text_model( config = WinMLEvaluationConfig( model_id="test/model", task="text-classification", - dataset=DatasetConfig(path="glue", name="mrpc"), + dataset=DatasetConfig(path="nyu-mll/glue", name="mrpc"), ) WinMLTextClassificationEvaluator(config, model).compute() @@ -847,7 +847,7 @@ def test_no_padding_without_tokenizer( config = WinMLEvaluationConfig( model_id="test/model", task="text-classification", - dataset=DatasetConfig(path="glue"), + dataset=DatasetConfig(path="nyu-mll/glue"), ) WinMLTextClassificationEvaluator(config, model).compute()