From 31bb863d83542980a1f89524358c25bb09bb9021 Mon Sep 17 00:00:00 2001 From: Zhenchao Ni Date: Fri, 31 Jul 2026 13:41:39 +0800 Subject: [PATCH] fix(datasets): use fully qualified HF dataset ids (#1262) `huggingface_hub` now requires repository ids in `namespace/name` form, so the legacy bare canonical ids no longer resolve. The default calibration dataset was still requesting `glue`, which fails with: ``` HfUriError: Invalid HF URI 'hf://datasets/glue@.../.huggingface.yaml'. Repository id must be 'namespace/name', got 'glue'. ``` This never surfaced as a hard failure. When `TextDataset` construction fails, `universal_calib_dataset` silently falls back to `RandomDataset`, so `winml quantize` still reported success while calibrating on random noise instead of real text. That silently degrades quantization accuracy for every text task routed through `TextDataset` (`text-classification`, `feature-extraction`, `fill-mask`, `sentence-similarity`, `next-sentence-prediction`, `zero-shot-classification`). The fix is to use fully qualified dataset ids: - `glue` -> `nyu-mll/glue` (calibration default in `datasets/text.py`) - `tweet_eval` -> `cardiffnlp/tweet_eval` (e2e testset, recipe configs, and the matching baseline cache keys) The baseline cache keys are derived from `dataset_config["path"]`, so renaming the path required renaming the keys to keep the cached entries reachable. The recorded metrics are unchanged, so no baselines needed recomputing. Everything else is example paths in help text and docstrings. Verified with `winml export` followed by `winml quantize` on `philschmid/tiny-bert-sst2-distilled` and `cross-encoder/nli-deberta-v3-small`. Before the fix both runs logged the `HfUriError` and fell back to `RandomDataset`; after the fix both load real MRPC sentence pairs. This also covers the zero-shot problem reported in #1261. `zero-shot-classification` routes through the same `TextDataset`, and MRPC is natively a sentence-pair dataset, so calibration now gets the premise/hypothesis style input those models expect. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../text-classification_fp16_config.json | 2 +- .../text-classification_w8a16_config.json | 2 +- scripts/e2e_eval/cache/baseline_cache.json | 8 ++++---- scripts/e2e_eval/testsets/models_with_acc.json | 2 +- src/winml/modelkit/commands/eval.py | 2 +- src/winml/modelkit/datasets/text.py | 11 ++++++++--- src/winml/modelkit/eval/config.py | 2 +- .../integration/datasets/test_text_classification.py | 10 +++++----- tests/unit/eval/test_eval.py | 6 +++--- 9 files changed, 25 insertions(+), 20 deletions(-) diff --git a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json index 8de0a638d..683080239 100644 --- a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json +++ b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_fp16_config.json @@ -53,7 +53,7 @@ "eval": { "task": "text-classification", "dataset": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "samples": 100, "columns_mapping": { diff --git a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json index eeb1fff26..cd55911db 100644 --- a/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json +++ b/examples/recipes/cardiffnlp_twitter-roberta-base-sentiment-latest/text-classification_w8a16_config.json @@ -70,7 +70,7 @@ "eval": { "task": "text-classification", "dataset": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "samples": 1000, "columns_mapping": { diff --git a/scripts/e2e_eval/cache/baseline_cache.json b/scripts/e2e_eval/cache/baseline_cache.json index 8acbbe305..7590060d4 100644 --- a/scripts/e2e_eval/cache/baseline_cache.json +++ b/scripts/e2e_eval/cache/baseline_cache.json @@ -9,7 +9,7 @@ "elapsed": 59.2, "command": "python.exe run_pytorch_baseline.py --model finbert --task text-classification --device cpu --num-samples 1000 --dataset finbert_dataset --split val" }, - "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|tweet_eval|sentiment||1000": { + "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|cardiffnlp/tweet_eval|sentiment||1000": { "status": "PASS", "metric": { "metric": "accuracy", @@ -17,7 +17,7 @@ "num_samples": 1000 }, "elapsed": 91.5, - "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 1000 --dataset tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"}" + "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 1000 --dataset cardiffnlp/tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"}" }, "distilbert/distilbert-base-uncased-finetuned-sst-2-english|text-classification|nyu-mll/glue|sst2||1000": { "status": "PASS", @@ -959,7 +959,7 @@ "elapsed": 52.0, "command": "python.exe run_pytorch_baseline.py --model finbert --task text-classification --device cpu --num-samples 100 --dataset finbert_dataset --split val --winml-metric-key accuracy" }, - "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|tweet_eval|sentiment||100": { + "cardiffnlp/twitter-roberta-base-sentiment-latest|text-classification|cardiffnlp/tweet_eval|sentiment||100": { "status": "PASS", "metric": { "metric": "accuracy", @@ -967,7 +967,7 @@ "num_samples": 100 }, "elapsed": 54.9, - "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 100 --dataset tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"} --winml-metric-key accuracy" + "command": "python.exe run_pytorch_baseline.py --model twitter-roberta-base-sentiment-latest --task text-classification --device cpu --num-samples 100 --dataset cardiffnlp/tweet_eval --dataset-config sentiment --columns-mapping {\"input_column\": \"text\"} --winml-metric-key accuracy" }, "distilbert/distilbert-base-uncased-finetuned-sst-2-english|text-classification|nyu-mll/glue|sst2||100": { "status": "PASS", diff --git a/scripts/e2e_eval/testsets/models_with_acc.json b/scripts/e2e_eval/testsets/models_with_acc.json index e21423161..03a136206 100644 --- a/scripts/e2e_eval/testsets/models_with_acc.json +++ b/scripts/e2e_eval/testsets/models_with_acc.json @@ -18,7 +18,7 @@ "group": "Top200", "priority": "P1", "dataset_config": { - "path": "tweet_eval", + "path": "cardiffnlp/tweet_eval", "name": "sentiment", "metric": "accuracy", "columns_mapping": { diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index 07f1af61c..68d0c3e1c 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -45,7 +45,7 @@ "dataset_path", type=str, default=None, - help="HF dataset path (e.g. 'imagenet-1k', 'glue'). " + help="HF dataset path (e.g. 'imagenet-1k', 'nyu-mll/glue'). " "If omitted, uses a default dataset for the task.", ) @click.option( diff --git a/src/winml/modelkit/datasets/text.py b/src/winml/modelkit/datasets/text.py index e44f115e2..f3f709d69 100644 --- a/src/winml/modelkit/datasets/text.py +++ b/src/winml/modelkit/datasets/text.py @@ -27,6 +27,11 @@ logger = logging.getLogger(__name__) +# HF requires fully qualified repository ids ("namespace/name"); the legacy +# canonical alias "glue" is no longer resolvable. +DEFAULT_TEXT_DATASET = "nyu-mll/glue" +DEFAULT_TEXT_DATASET_SUBSET = "mrpc" + class TextDataset(BaseTaskDataset): """Dataset for text tasks with universal tokenization. @@ -56,7 +61,7 @@ def __init__( Args: model_name: HuggingFace model identifier - dataset_name: Dataset name (default: glue) + dataset_name: Dataset name (default: nyu-mll/glue) max_samples: Maximum samples (None = use all) data_split: Dataset split (default: train) max_length: Sequence length (default: from io_config or 128) @@ -85,8 +90,8 @@ def __init__( def _get_default_dataset(self) -> None: """Set default dataset if none specified.""" if self._dataset_name is None: - self._dataset_name = "glue" - self._config["subset"] = self._config.get("subset", "mrpc") + self._dataset_name = DEFAULT_TEXT_DATASET + self._config["subset"] = self._config.get("subset", DEFAULT_TEXT_DATASET_SUBSET) self._data_split = self._data_split or "train" def _resolve_max_length(self) -> None: diff --git a/src/winml/modelkit/eval/config.py b/src/winml/modelkit/eval/config.py index 1717ccdb5..03944010a 100644 --- a/src/winml/modelkit/eval/config.py +++ b/src/winml/modelkit/eval/config.py @@ -20,7 +20,7 @@ class DatasetConfig: """Dataset configuration, aligned with HF load_dataset() API. Attributes: - path: HF dataset path (e.g., "imagenet-1k", "glue"). + path: HF dataset path (e.g., "imagenet-1k", "nyu-mll/glue"). name: Config name for multi-config datasets (e.g., "mrpc"). split: Dataset split. samples: Number of samples to evaluate. diff --git a/tests/integration/datasets/test_text_classification.py b/tests/integration/datasets/test_text_classification.py index 9559d4006..d1a291d63 100644 --- a/tests/integration/datasets/test_text_classification.py +++ b/tests/integration/datasets/test_text_classification.py @@ -44,7 +44,7 @@ def test_default_seq_len_constant(self): assert TextDataset.DEFAULT_SEQ_LEN == 128 def test_default_dataset_glue_mrpc(self): - """Test default dataset is glue/mrpc when none specified.""" + """Test default dataset is nyu-mll/glue with the mrpc subset.""" from winml.modelkit.datasets import TextDataset dataset = TextDataset( @@ -52,7 +52,7 @@ def test_default_dataset_glue_mrpc(self): max_samples=5, ) - assert dataset.dataset_name == "glue" + assert dataset.dataset_name == "nyu-mll/glue" assert dataset.data_split == "train" def test_explicit_dataset_name(self): @@ -61,13 +61,13 @@ def test_explicit_dataset_name(self): dataset = TextDataset( model_name="bert-base-uncased", - dataset_name="glue", + dataset_name="nyu-mll/glue", data_split="validation", max_samples=5, subset="sst2", ) - assert dataset.dataset_name == "glue" + assert dataset.dataset_name == "nyu-mll/glue" assert dataset.data_split == "validation" def test_max_samples_limits_dataset_size(self): @@ -274,7 +274,7 @@ def test_single_sentence_detection_sst2(self): # GLUE/SST2 is a single sentence task dataset = TextDataset( model_name="bert-base-uncased", - dataset_name="glue", + dataset_name="nyu-mll/glue", data_split="train", max_samples=5, subset="sst2", diff --git a/tests/unit/eval/test_eval.py b/tests/unit/eval/test_eval.py index 6c5bb469b..dbf93b72e 100644 --- a/tests/unit/eval/test_eval.py +++ b/tests/unit/eval/test_eval.py @@ -752,7 +752,7 @@ def _fake_compute(**kw): model_id="test/model", task="text-classification", dataset=DatasetConfig( - path="glue", + path="nyu-mll/glue", name="mrpc", columns_mapping={"input_column": "sentence1", "second_input_column": "sentence2"}, ), @@ -803,7 +803,7 @@ def test_sets_padding_for_text_model( config = WinMLEvaluationConfig( model_id="test/model", task="text-classification", - dataset=DatasetConfig(path="glue", name="mrpc"), + dataset=DatasetConfig(path="nyu-mll/glue", name="mrpc"), ) WinMLTextClassificationEvaluator(config, model).compute() @@ -847,7 +847,7 @@ def test_no_padding_without_tokenizer( config = WinMLEvaluationConfig( model_id="test/model", task="text-classification", - dataset=DatasetConfig(path="glue"), + dataset=DatasetConfig(path="nyu-mll/glue"), ) WinMLTextClassificationEvaluator(config, model).compute()