From 1c88af946a13ca8be4b3a1ebe6397d0cc494b9b5 Mon Sep 17 00:00:00 2001 From: Satya Phanindra Kumar Kalaga <51989959+phanindra-max@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:07:27 -0400 Subject: [PATCH] Revert "fix: default all LLM backends to deterministic greedy decoding" (#428) * Revert "fix: default all LLM backends to deterministic greedy decoding" * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * fix: kwargs for gemini models Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- laiser/config.py | 12 -- laiser/llm_methods.py | 14 +- laiser/llm_models/anthropic.py | 15 -- laiser/llm_models/gemini.py | 34 +-- laiser/llm_models/hugging_face_llm.py | 65 +----- laiser/llm_models/llama_cpp_handler.py | 30 +-- laiser/llm_models/llm_router.py | 45 +--- laiser/llm_models/openai.py | 19 +- laiser/services.py | 8 - laiser/skill_extractor_refactored.py | 18 +- paper.md | 8 +- pytest.ini | 3 +- tests/test_determinism.py | 274 ------------------------- 13 files changed, 27 insertions(+), 518 deletions(-) delete mode 100644 tests/test_determinism.py diff --git a/laiser/config.py b/laiser/config.py index 0f76a79..03945a7 100644 --- a/laiser/config.py +++ b/laiser/config.py @@ -92,18 +92,6 @@ MAX_NEW_TOKENS = 1000 GENERATION_SEED = 42 -# Deterministic decoding defaults. -# -# LAiSER defaults every LLM backend to greedy decoding so that repeated runs over -# the same input yield the same generation. DEFAULT_TEMPERATURE = 0.0 selects the -# highest-probability token at each step; DEFAULT_TOP_P = 1.0 leaves nucleus -# filtering inactive so that temperature alone governs the decoding behaviour. -# GENERATION_SEED is supplied to every backend that accepts a sampling seed, so -# that runs remain reproducible even when a caller deliberately raises the -# temperature above zero. Callers may override both via SkillExtractorRefactored. -DEFAULT_TEMPERATURE = float(os.getenv("LAISER_TEMPERATURE", "0.0")) -DEFAULT_TOP_P = float(os.getenv("LAISER_TOP_P", "1.0")) - # SCQF Level Descriptors SCQF_LEVELS: Dict[int, str] = { 1: "Basic awareness of simple concepts.", diff --git a/laiser/llm_methods.py b/laiser/llm_methods.py index f3ae09e..a2be0a6 100644 --- a/laiser/llm_methods.py +++ b/laiser/llm_methods.py @@ -75,7 +75,6 @@ VLLM_AVAILABLE = False SamplingParams = None -from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P, GENERATION_SEED from laiser.utils import get_top_esco_skills # Import llm_router with error handling @@ -176,7 +175,7 @@ def get_completion_batch(queries: list, model, tokenizer, batch_size=2) -> list: generated_ids = model.generate( **model_inputs, max_new_tokens=1000, - do_sample=DEFAULT_TEMPERATURE > 0.0, + do_sample=True, pad_token_id=tokenizer.eos_token_id, ) @@ -260,14 +259,14 @@ def get_completion(input_text, text_columns, input_type, model, tokenizer) -> st generated_ids = model.generate( **model_inputs, max_new_tokens=1000, - do_sample=DEFAULT_TEMPERATURE > 0.0, + do_sample=True, pad_token_id=tokenizer.eos_token_id, ) generated_ids = model.generate( **model_inputs, max_new_tokens=1000, - do_sample=DEFAULT_TEMPERATURE > 0.0, + do_sample=True, pad_token_id=tokenizer.eos_token_id, ) decoded = tokenizer.decode(generated_ids[0], skip_special_tokens=False) @@ -477,12 +476,7 @@ def vllm_generate( result = [] - sampling_params = SamplingParams( - max_tokens=1000, - temperature=DEFAULT_TEMPERATURE, - top_p=DEFAULT_TOP_P, - seed=GENERATION_SEED, - ) + sampling_params = SamplingParams(max_tokens=1000, seed=42) for i in range(0, len(queries), batch_size): prompts = [ diff --git a/laiser/llm_models/anthropic.py b/laiser/llm_models/anthropic.py index 534141e..ce85a4b 100644 --- a/laiser/llm_models/anthropic.py +++ b/laiser/llm_models/anthropic.py @@ -4,8 +4,6 @@ import requests -from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P - _CODE_FENCE_RE = re.compile(r"```(?:json)?|```", re.IGNORECASE) @@ -23,17 +21,7 @@ def anthropic_generate( model: str = "claude-haiku-4-5-20251001", timeout: int = 60, max_tokens: int = 4096, - temperature: float = DEFAULT_TEMPERATURE, - top_p: float = DEFAULT_TOP_P, ) -> str: - """Generate text with the Anthropic Messages API. - - Decoding is deterministic by default: ``temperature`` defaults to - ``config.DEFAULT_TEMPERATURE`` (0.0, i.e. greedy decoding). ``top_p`` is - only transmitted when it departs from 1.0, because the Messages API - discourages setting both ``temperature`` and ``top_p`` in one request. - The API does not accept a sampling seed. - """ api_key = api_key or os.getenv("ANTHROPIC_API_KEY") if not api_key: raise ValueError("Anthropic API key not provided") @@ -47,11 +35,8 @@ def anthropic_generate( payload = { "model": model, "max_tokens": max_tokens, - "temperature": temperature, "messages": [{"role": "user", "content": prompt}], } - if top_p is not None and top_p != 1.0: - payload["top_p"] = top_p try: resp = requests.post(url, headers=headers, json=payload, timeout=timeout) diff --git a/laiser/llm_models/gemini.py b/laiser/llm_models/gemini.py index 56e18d7..9be0a32 100644 --- a/laiser/llm_models/gemini.py +++ b/laiser/llm_models/gemini.py @@ -9,19 +9,11 @@ from google.genai import errors as genai_errors from google.genai import types -from laiser.config import DEFAULT_TEMPERATURE, GENERATION_SEED - DEFAULT_GEMINI_MODEL = os.getenv("LAISER_GEMINI_MODEL", "gemini-2.5-flash") DEFAULT_GEMINI_TIMEOUT = float(os.getenv("LAISER_GEMINI_TIMEOUT", "60")) DEFAULT_MAX_OUTPUT_TOKENS = int(os.getenv("LAISER_GEMINI_MAX_OUTPUT_TOKENS", "1000")) -# Older google-genai releases do not expose a sampling seed on -# GenerateContentConfig. Detect support once so that requesting a seed never -# breaks generation on those versions. -_SUPPORTS_SEED = "seed" in getattr(types.GenerateContentConfig, "model_fields", {}) - - class GeminiAPI: """Small wrapper to keep backward compatibility with older imports.""" @@ -31,8 +23,6 @@ def __init__( model_name: Optional[str] = None, timeout: float = DEFAULT_GEMINI_TIMEOUT, max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS, - temperature: float = DEFAULT_TEMPERATURE, - seed: Optional[int] = GENERATION_SEED, ): if not api_key: raise ValueError("Gemini API key is required") @@ -40,8 +30,6 @@ def __init__( self.model_name = model_name or DEFAULT_GEMINI_MODEL self.timeout = timeout self.max_output_tokens = max_output_tokens - self.temperature = temperature - self.seed = seed self.client = genai.Client(api_key=api_key) def generate(self, prompt: str) -> str: @@ -55,9 +43,7 @@ def generate_with_config( response_schema: Optional[Any] = None, ) -> str: try: - config_kwargs = {"temperature": self.temperature} - if self.seed is not None and _SUPPORTS_SEED: - config_kwargs["seed"] = self.seed + config_kwargs = {"temperature": 0.0, "max_output_tokens": self.max_output_tokens} if response_mime_type: config_kwargs["response_mime_type"] = response_mime_type if response_schema is not None: @@ -90,23 +76,9 @@ def gemini_generate( max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS, response_mime_type: Optional[str] = None, response_schema: Optional[Any] = None, - temperature: float = DEFAULT_TEMPERATURE, - seed: Optional[int] = GENERATION_SEED, ) -> str: - """Send `prompt` to Gemini and return generated text. - - Decoding is deterministic by default: ``temperature`` defaults to - ``config.DEFAULT_TEMPERATURE`` (0.0, i.e. greedy decoding) and - ``config.GENERATION_SEED`` is supplied wherever the installed google-genai - release accepts a sampling seed. - """ - client = GeminiAPI( - api_key=api_key, - model_name=model_name, - timeout=timeout, - temperature=temperature, - seed=seed, - ) + """Send `prompt` to Gemini and return generated text.""" + client = GeminiAPI(api_key=api_key, model_name=model_name, timeout=timeout) return client.generate_with_config( prompt, response_mime_type=response_mime_type, diff --git a/laiser/llm_models/hugging_face_llm.py b/laiser/llm_models/hugging_face_llm.py index 09036c8..3408460 100644 --- a/laiser/llm_models/hugging_face_llm.py +++ b/laiser/llm_models/hugging_face_llm.py @@ -49,11 +49,8 @@ [1.0.0] 6/30/2025 Anket Patil Modularize LLM generation logic for transformers and vLLM """ -import torch from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig -from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P, GENERATION_SEED - try: from vllm import SamplingParams @@ -63,24 +60,7 @@ SamplingParams = None # Optional fallback -def llm_generate( - prompt: str, - tokenizer, - model, - model_id: str, - use_gpu: bool, - temperature: float = DEFAULT_TEMPERATURE, - seed: int = GENERATION_SEED, -): - """Generate text with a local HuggingFace Transformers model. - - Decoding is deterministic by default. A temperature of 0.0 disables - sampling entirely (``do_sample=False``, i.e. greedy decoding), which also - prevents a checkpoint's bundled ``generation_config.json`` from silently - re-enabling sampling. When a caller raises the temperature above zero, - ``seed`` is applied through ``torch.manual_seed`` so the run stays - reproducible. - """ +def llm_generate(prompt: str, tokenizer, model, model_id: str, use_gpu: bool): if tokenizer is None or model is None: quantization_config = BitsAndBytesConfig(load_in_8bit=True) @@ -91,50 +71,23 @@ def llm_generate( inputs = tokenizer(prompt, return_tensors="pt").to(model.device) - do_sample = temperature is not None and temperature > 0.0 - if seed is not None: - torch.manual_seed(seed) - - generation_kwargs = { - "max_new_tokens": 100, - "pad_token_id": tokenizer.pad_token_id, - "eos_token_id": tokenizer.eos_token_id, - "do_sample": do_sample, - } - if do_sample: - generation_kwargs["temperature"] = temperature - generation_kwargs["top_p"] = DEFAULT_TOP_P - - outputs = model.generate(**inputs, **generation_kwargs) + outputs = model.generate( + **inputs, + max_new_tokens=100, + pad_token_id=tokenizer.pad_token_id, + eos_token_id=tokenizer.eos_token_id, + ) return tokenizer.decode(outputs[0], skip_special_tokens=True) -def llm_generate_vllm( - prompt, - llm, - temperature: float = DEFAULT_TEMPERATURE, - seed: int = GENERATION_SEED, - max_tokens: int = 200, -): - """Generate text with a local vLLM engine. - - Decoding is deterministic by default: ``temperature`` defaults to - ``config.DEFAULT_TEMPERATURE`` (0.0, i.e. greedy decoding) rather than to - vLLM's own default of 1.0, and ``config.GENERATION_SEED`` is passed so that - runs remain reproducible if a caller raises the temperature. - """ +def llm_generate_vllm(prompt, llm): if not VLLM_AVAILABLE: raise ImportError( "vLLM is not installed. Please install it to use this function." ) - sampling_params = SamplingParams( - max_tokens=max_tokens, - temperature=temperature, - top_p=DEFAULT_TOP_P, - seed=seed, - ) + sampling_params = SamplingParams(max_tokens=200, seed=42) result = llm.generate([prompt], sampling_params=sampling_params) raw_text = result[0].outputs[0].text.strip() return raw_text diff --git a/laiser/llm_models/llama_cpp_handler.py b/laiser/llm_models/llama_cpp_handler.py index ca649f4..e6172a9 100644 --- a/laiser/llm_models/llama_cpp_handler.py +++ b/laiser/llm_models/llama_cpp_handler.py @@ -1,31 +1,15 @@ import gc -import inspect import os import re from pathlib import Path from typing import Any, List, Optional -from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P, GENERATION_SEED - try: from llama_cpp import Llama # type: ignore except ImportError: # pragma: no cover Llama = None -def _llama_accepts_seed(llama: Any) -> bool: - """Report whether this llama-cpp-python build accepts a per-call seed. - - The ``seed`` argument to ``create_chat_completion`` was added in later - releases; probing the signature keeps deterministic seeding from raising a - TypeError on older installations. - """ - try: - return "seed" in inspect.signature(llama.create_chat_completion).parameters - except (TypeError, ValueError): - return False - - def _strip_fences(text: str) -> str: if not text: return "" @@ -43,7 +27,7 @@ def __init__( n_ctx: int = 4096, n_threads: Optional[int] = None, n_gpu_layers: int = -1, - temperature: float = DEFAULT_TEMPERATURE, + temperature: float = 0.2, chat_format: str = "chatml", ): if Llama is None: @@ -121,16 +105,9 @@ def llama_cpp_chat( system: str = "You are a helpful assistant that outputs in JSON.", max_tokens: Optional[int] = None, stop: Optional[List[str]] = None, - temperature: Optional[float] = DEFAULT_TEMPERATURE, - seed: Optional[int] = GENERATION_SEED, + temperature: Optional[float] = None, ) -> str: - """Generate a chat completion with a local llama.cpp model. - Decoding is deterministic by default: ``temperature`` defaults to - ``config.DEFAULT_TEMPERATURE`` (0.0, i.e. greedy decoding) and - ``config.GENERATION_SEED`` is supplied so that runs remain reproducible if - a caller raises the temperature. - """ if llama is None: raise ValueError( "llama is None; expected an initialized llama_cpp.Llama instance." @@ -148,9 +125,6 @@ def llama_cpp_chat( kwargs["stop"] = stop if temperature is not None: kwargs["temperature"] = temperature - kwargs["top_p"] = DEFAULT_TOP_P - if seed is not None and _llama_accepts_seed(llama): - kwargs["seed"] = seed resp = llama.create_chat_completion(**kwargs) return _strip_fences(resp["choices"][0]["message"]["content"]) diff --git a/laiser/llm_models/llm_router.py b/laiser/llm_models/llm_router.py index 6ac47ff..341a9df 100644 --- a/laiser/llm_models/llm_router.py +++ b/laiser/llm_models/llm_router.py @@ -53,7 +53,6 @@ import torch -from laiser.config import DEFAULT_TEMPERATURE, GENERATION_SEED from laiser.exceptions import LAiSERError from laiser.llm_models.llama_cpp_handler import llama_cpp_chat from laiser.llm_models.model_loader import load_model_from_transformer, load_model_from_vllm @@ -89,35 +88,12 @@ def llm_generate_vllm(*args, **kwargs): class LLMRouter: - def __init__( - self, - model_id: str, - use_gpu: bool, - hf_token=None, - api_key=None, - backend=None, - temperature: float = DEFAULT_TEMPERATURE, - seed=GENERATION_SEED, - ): - """Route generation requests to the configured LLM backend. - - Parameters - ---------- - temperature : float - Decoding temperature applied to every backend. Defaults to - ``config.DEFAULT_TEMPERATURE`` (0.0), which selects greedy, - deterministic decoding. - seed : int or None - Sampling seed forwarded to every backend that accepts one, so that - runs remain reproducible if ``temperature`` is raised above zero. - """ + def __init__(self, model_id: str, use_gpu: bool, hf_token=None, api_key=None, backend=None): self.model_id = model_id self.use_gpu = use_gpu self.hf_token = hf_token self.api_key = api_key self.backend = backend - self.temperature = temperature - self.seed = seed self.llm = None self.model = None @@ -128,32 +104,19 @@ def __init__( # ---------------- ROUTER ---------------- def generate(self, prompt: str, **kwargs): - """Dispatch a prompt to the active backend using deterministic decoding. - - The router's ``temperature`` and ``seed`` are forwarded to every - backend, so decoding behaviour is identical regardless of which one is - selected. Explicit values in ``kwargs`` take precedence. - """ - kwargs.setdefault("temperature", self.temperature) - if self.model_id == "gemini": - kwargs.setdefault("seed", self.seed) return gemini_generate(prompt, self.api_key, **kwargs) if self.model_id == "openai": - return openai_generate(prompt, self.api_key, temperature=kwargs["temperature"]) + return openai_generate(prompt, self.api_key) # If a local GGUF model was loaded with llama-cpp-python, use it if self.backend == "llama_cpp": print("LLMRouter: routing request to llama_cpp backend") - return llama_cpp_chat( - prompt, self.llm, temperature=kwargs["temperature"], seed=self.seed - ) + return llama_cpp_chat(prompt, self.llm) print("LLMRouter: routing request to vLLM/transformer backend") - return llm_generate_vllm( - prompt, self.llm, temperature=kwargs["temperature"], seed=self.seed - ) + return llm_generate_vllm(prompt, self.llm) # ---------------- INIT ---------------- def _initialize_components(self): diff --git a/laiser/llm_models/openai.py b/laiser/llm_models/openai.py index 00186af..7da53a1 100644 --- a/laiser/llm_models/openai.py +++ b/laiser/llm_models/openai.py @@ -4,8 +4,6 @@ import requests -from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P - _CODE_FENCE_RE = re.compile(r"```(?:json)?|```", re.IGNORECASE) @@ -22,17 +20,7 @@ def openai_generate( api_key: Optional[str] = None, model: str = "gpt-4.1-mini", timeout: int = 60, - temperature: float = DEFAULT_TEMPERATURE, - top_p: float = DEFAULT_TOP_P, ) -> str: - """Generate text with the OpenAI Responses API. - - Decoding is deterministic by default: ``temperature`` defaults to - ``config.DEFAULT_TEMPERATURE`` (0.0, i.e. greedy decoding) and ``top_p`` - defaults to ``config.DEFAULT_TOP_P`` (1.0, nucleus filtering inactive). - The Responses API does not accept a sampling seed, so reproducibility on - this backend rests on greedy decoding alone. - """ api_key = api_key or os.getenv("OPENAI_API_KEY") if not api_key: raise ValueError("OpenAI API key not provided") @@ -43,12 +31,7 @@ def openai_generate( "Content-Type": "application/json", } - payload = { - "model": model, - "input": prompt, - "temperature": temperature, - "top_p": top_p, - } + payload = {"model": model, "input": prompt} try: resp = requests.post(url, headers=headers, json=payload, timeout=timeout) diff --git a/laiser/services.py b/laiser/services.py index a2c9747..b752fda 100644 --- a/laiser/services.py +++ b/laiser/services.py @@ -17,9 +17,7 @@ COMBINED_EXTRACTION_PROMPT, DEFAULT_BATCH_SIZE, DEFAULT_SIMILARITY_THRESHOLDS, - DEFAULT_TEMPERATURE, DEFAULT_TOP_K, - GENERATION_SEED, KSA_DETAILS_PROMPT, KSA_EXTRACTION_PROMPT, KT_FROM_SKILLS_PROMPT, @@ -630,8 +628,6 @@ def __init__( api_key: Optional[str] = None, use_gpu: Optional[bool] = None, backend: Optional[str] = None, - temperature: float = DEFAULT_TEMPERATURE, - seed: Optional[int] = GENERATION_SEED, ): self.model_id = model_id @@ -639,8 +635,6 @@ def __init__( self.api_key = api_key self.use_gpu = use_gpu if use_gpu is not None else torch.cuda.is_available() self.backend = backend - self.temperature = temperature - self.seed = seed self.llm = None self.tokenizer = None self.model = None @@ -669,8 +663,6 @@ def __init__( self.hf_token, self.api_key, backend=self.backend, - temperature=self.temperature, - seed=self.seed, ) # Initialize skills FAISS index diff --git a/laiser/skill_extractor_refactored.py b/laiser/skill_extractor_refactored.py index f7c212a..0d90984 100644 --- a/laiser/skill_extractor_refactored.py +++ b/laiser/skill_extractor_refactored.py @@ -61,7 +61,7 @@ import pandas as pd -from laiser.config import DEFAULT_BATCH_SIZE, DEFAULT_TEMPERATURE, GENERATION_SEED +from laiser.config import DEFAULT_BATCH_SIZE from laiser.services import SkillExtractionService @@ -106,8 +106,6 @@ def __init__( api_key: Optional[str] = None, use_gpu: Optional[bool] = None, backend: Optional[str] = None, - temperature: float = DEFAULT_TEMPERATURE, - seed: Optional[int] = GENERATION_SEED, ): """ Initialize the skill extractor. @@ -124,18 +122,6 @@ def __init__( Whether to use GPU for model inference backend : str, optional Backend to use for LLM inference (e.g., "llama_cpp", "huggingface", "openai", "gemini") - temperature : float, optional - Decoding temperature applied to whichever backend is selected. - Defaults to ``config.DEFAULT_TEMPERATURE`` (0.0), which selects - greedy decoding and makes repeated runs over the same input - reproducible. Raise it only when sampled variation is wanted. - seed : int, optional - Sampling seed forwarded to every backend that accepts one (vLLM, - llama.cpp, Gemini, and local Transformers). Defaults to - ``config.GENERATION_SEED``. Set to ``None`` to leave the seed - unset. The OpenAI Responses API and the Anthropic Messages API do - not accept a seed, so on those backends reproducibility rests on - greedy decoding alone. """ # Initialize service layer self.skill_service = SkillExtractionService( @@ -144,8 +130,6 @@ def __init__( hf_token=hf_token, use_gpu=use_gpu, backend=backend, - temperature=temperature, - seed=seed, ) def extract_concepts( diff --git a/paper.md b/paper.md index faf6f9e..4b2eccb 100644 --- a/paper.md +++ b/paper.md @@ -45,13 +45,9 @@ LAiSER differs from these tools in several key respects. First, it uses LLMs for LAiSER's architecture is organized into three loosely coupled layers: data access, skill extraction, and taxonomy alignment. This separation was chosen to allow each component to evolve independently and to support diverse deployment scenarios without requiring changes to the overall pipeline. -The extraction layer posed the most significant design trade-off. Training a custom NER model, as Nesta's library does, would yield fast inference but would require substantial labeled data for each new domain (syllabi, credentials, job postings). Instead, LAiSER delegates extraction to LLMs via a prompt engineering approach, trading some inference speed for broad domain generalizability without retraining. To mitigate vendor lock-in and hardware constraints, the extraction layer abstracts over multiple backends: vLLM for GPU clusters, HuggingFace Transformers and llama.cpp for local models, and the Gemini and OpenAI APIs for cloud-based inference. The active backend is resolved at initialization from the available hardware and credentials, so the same extraction code runs unchanged on a GPU cluster, a laptop, or a hosted API. +The extraction layer posed the most significant design trade-off. Training a custom NER model, as Nesta's library does, would yield fast inference but would require substantial labeled data for each new domain (syllabi, credentials, job postings). Instead, LAiSER delegates extraction to LLMs via a prompt engineering approach, trading some inference speed for broad domain generalizability without retraining. To mitigate vendor lock-in and hardware constraints, the extraction layer abstracts over multiple backends (vLLM for GPU clusters, HuggingFace Transformers for local models, llama.cpp for local GGUF models, and the Gemini and OpenAI APIs for cloud-based inference), with automatic fallback when a backend is unavailable. -For taxonomy alignment, the framework embeds both the extracted phrases and the taxonomy entries with a sentence-transformer model [@Reimers2019] and retrieves matches from a precomputed FAISS index [@Johnson2019]. The index is a flat inner-product index over L2-normalized embeddings, which makes retrieval exhaustive and exact rather than approximate. Exactness was a deliberate choice over an approximate index: at the scale of ESCO's 13,000+ skill entries an exhaustive scan is inexpensive relative to the cost of LLM inference in the preceding stage, and it guarantees that a given query returns the same neighbor and the same similarity score on every run, which is a precondition for the reproducibility properties described below. Because retrieval is encapsulated behind an index manager, an approximate index can be substituted without changes to the alignment logic should taxonomy size later make exhaustive search impractical. The system accepts pandas DataFrames as input and produces structured output including raw extracted skills, taxonomy-aligned canonical skills, taxonomy identifiers (e.g., ESCO codes), semantic similarity scores, and optional SCQF proficiency levels. - -Because large language models are stochastic samplers, identical inputs can yield different generations across runs, and different backends can yield different generations from the same input. LAiSER addresses this in three ways. First, decoding is deterministic by default: every backend is configured for greedy decoding at temperature zero, and a fixed generation seed is supplied to each backend that accepts one. Both the temperature and the seed are exposed as constructor arguments, so sampled variation is available when a user wants it but is never the silent default. Second, the taxonomy alignment stage acts as a normalizing projection. Raw generations are embedded with a fixed sentence-transformer model and matched by exact inner-product search against a precomputed taxonomy index, so surface variation in the generated text is collapsed onto a closed, finite vocabulary of canonical entries with stable identifiers. Distinct phrasings of the same underlying competency therefore converge to the same aligned record, and the aligned output is more stable than the raw generations that produce it. Third, all backends terminate in a single output schema comprising the raw skill, the aligned canonical skill, its taxonomy identifier and source, and the similarity score, which makes results directly comparable across models and allows cross-model agreement to be measured on the aligned columns. The test suite asserts both properties: that each backend's decoding defaults select greedy decoding, and that repeated alignment of a fixed input yields an identical frame including similarity scores. - -Residual variability remains and is not eliminated by these measures. Alignment normalizes how a skill is expressed, but it cannot recover a skill that a model failed to emit, so the number and identity of extracted skills can still differ across models. We therefore recommend that users pin the model identifier, the seed, and the similarity threshold when reporting results, and treat cross-model agreement as an empirical quantity to be measured rather than assumed. +For taxonomy alignment, the framework embeds extracted phrases and taxonomy entries with a sentence-transformer model [@Reimers2019] and retrieves matches from a precomputed FAISS index using a flat inner-product index (exact search over normalized embeddings). This choice keeps alignment behavior stable and reproducible at ESCO scale while remaining inexpensive relative to LLM inference. The system accepts pandas DataFrames as input and produces structured output including raw extracted skills, taxonomy-aligned canonical skills, taxonomy identifiers (e.g., ESCO codes), semantic similarity scores, and optional SCQF proficiency levels. # Research impact statement diff --git a/pytest.ini b/pytest.ini index c230a14..8d58ba0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,5 +5,4 @@ markers = library: entire library llm_cloud: cloud llms test openai: openai - anthropic: anthropic - determinism: decoding determinism and alignment stability \ No newline at end of file + anthropic: anthropic \ No newline at end of file diff --git a/tests/test_determinism.py b/tests/test_determinism.py deleted file mode 100644 index f089dab..0000000 --- a/tests/test_determinism.py +++ /dev/null @@ -1,274 +0,0 @@ -""" -Determinism and cross-backend consistency tests. - -These tests answer JOSS review issue #424: LLMs are stochastic samplers, so the -same input can yield different generations across runs, and different backends -can yield different generations from the same input. LAiSER addresses this by -(a) defaulting every backend to greedy decoding, and (b) projecting raw -generations onto a fixed taxonomy vocabulary in the alignment stage. - -The tests below verify both halves without requiring a GPU, a model download, -or a paid API call: - -- Decoding defaults are asserted directly against each backend's signature and, - for the HTTP backends, against the request payload that would be sent. -- Alignment stability is asserted by running the real alignment path repeatedly - over a fixed input and requiring byte-identical output. -""" - -import importlib -import inspect - -import pandas as pd -import pytest - -from laiser.config import DEFAULT_TEMPERATURE, DEFAULT_TOP_P, GENERATION_SEED - - -def _import_or_skip(module_path): - """Import a backend module, skipping if its optional dependency is absent. - - Backends such as vLLM, torch, google-genai, and llama-cpp-python are - optional extras, so a bare CPU install cannot import all of them. This - helper skips rather than fails in that case. It is used in preference to - pytest.importorskip because the latter re-raises ImportErrors that are not - ModuleNotFoundError, which is exactly the shape raised by - ``from google import genai`` when google-genai is not installed. - """ - try: - return importlib.import_module(module_path) - except ImportError as e: - pytest.skip(f"{module_path} unavailable in this environment: {e}") - - -# --------------------------------------------------------------------------- -# 1. Decoding defaults -# --------------------------------------------------------------------------- - - -def test_config_defaults_are_deterministic(): - """The shipped configuration must select greedy decoding.""" - assert DEFAULT_TEMPERATURE == 0.0, "Default decoding must be greedy (temperature 0.0)" - assert DEFAULT_TOP_P == 1.0, "Nucleus filtering must be inactive by default" - assert GENERATION_SEED is not None, "A default generation seed must be defined" - - -@pytest.mark.parametrize( - "module_path, func_name", - [ - ("laiser.llm_models.openai", "openai_generate"), - ("laiser.llm_models.anthropic", "anthropic_generate"), - ("laiser.llm_models.gemini", "gemini_generate"), - ("laiser.llm_models.hugging_face_llm", "llm_generate"), - ("laiser.llm_models.hugging_face_llm", "llm_generate_vllm"), - ("laiser.llm_models.llama_cpp_handler", "llama_cpp_chat"), - ], -) -def test_backend_defaults_to_zero_temperature(module_path, func_name): - """Every backend entry point must default to temperature 0.0. - - This is the guard against regression: adding a backend that inherits its - provider's default temperature (typically 1.0) will fail here. - """ - module = _import_or_skip(module_path) - func = getattr(module, func_name) - params = inspect.signature(func).parameters - - assert "temperature" in params, f"{func_name} does not expose a temperature parameter" - assert params["temperature"].default == DEFAULT_TEMPERATURE, ( - f"{func_name} defaults to temperature {params['temperature'].default}, " - f"expected {DEFAULT_TEMPERATURE}" - ) - - -@pytest.mark.parametrize( - "module_path, func_name", - [ - ("laiser.llm_models.gemini", "gemini_generate"), - ("laiser.llm_models.hugging_face_llm", "llm_generate"), - ("laiser.llm_models.hugging_face_llm", "llm_generate_vllm"), - ("laiser.llm_models.llama_cpp_handler", "llama_cpp_chat"), - ], -) -def test_seedable_backends_default_to_the_configured_seed(module_path, func_name): - """Backends that accept a sampling seed must default to GENERATION_SEED. - - The OpenAI Responses API and the Anthropic Messages API are excluded because - neither accepts a seed; on those backends reproducibility rests on greedy - decoding alone. - """ - module = _import_or_skip(module_path) - params = inspect.signature(getattr(module, func_name)).parameters - - assert "seed" in params, f"{func_name} does not expose a seed parameter" - assert params["seed"].default == GENERATION_SEED - - -def test_openai_payload_carries_deterministic_decoding(monkeypatch): - """The OpenAI request body must actually contain temperature 0.0.""" - openai_backend = _import_or_skip("laiser.llm_models.openai") - - captured = {} - - class _Response: - status_code = 200 - - def raise_for_status(self): - return None - - def json(self): - return {"output_text": "ok"} - - def _fake_post(url, headers=None, json=None, timeout=None): - captured.update(json or {}) - return _Response() - - monkeypatch.setattr(openai_backend.requests, "post", _fake_post) - openai_backend.openai_generate("prompt", api_key="test-key") - - assert captured["temperature"] == DEFAULT_TEMPERATURE - assert captured["top_p"] == DEFAULT_TOP_P - - -def test_anthropic_payload_carries_deterministic_decoding(monkeypatch): - """The Anthropic request body must actually contain temperature 0.0.""" - anthropic_backend = _import_or_skip("laiser.llm_models.anthropic") - - captured = {} - - class _Response: - status_code = 200 - - def raise_for_status(self): - return None - - def json(self): - return {"content": [{"type": "text", "text": "ok"}]} - - def _fake_post(url, headers=None, json=None, timeout=None): - captured.update(json or {}) - return _Response() - - monkeypatch.setattr(anthropic_backend.requests, "post", _fake_post) - anthropic_backend.anthropic_generate("prompt", api_key="test-key") - - assert captured["temperature"] == DEFAULT_TEMPERATURE - - -def test_router_forwards_deterministic_decoding(monkeypatch): - """LLMRouter must pass its temperature and seed through to the backend.""" - router_module = _import_or_skip("laiser.llm_models.llm_router") - - captured = {} - - def _fake_vllm(prompt, llm, temperature=None, seed=None, max_tokens=200): - captured["temperature"] = temperature - captured["seed"] = seed - return "ok" - - monkeypatch.setattr(router_module, "llm_generate_vllm", _fake_vllm) - monkeypatch.setattr(router_module.LLMRouter, "_initialize_components", lambda self: None) - - router = router_module.LLMRouter(model_id="local-model", use_gpu=False) - router.generate("prompt") - - assert captured["temperature"] == DEFAULT_TEMPERATURE - assert captured["seed"] == GENERATION_SEED - - -# --------------------------------------------------------------------------- -# 2. Alignment stability -# --------------------------------------------------------------------------- - - -RAW_SKILLS = [ - "Python programming", - "python programming skills", - "Experience programming in Python", - "statistical analysis", - "Data visualization techniques", - "machine learning model development", -] - - -@pytest.fixture(scope="module") -def alignment_service(): - data_access = _import_or_skip("laiser.data_access") - services = _import_or_skip("laiser.services") - DataAccessLayer = data_access.DataAccessLayer - FAISSIndexManager = data_access.FAISSIndexManager - SkillAlignmentService = services.SkillAlignmentService - - da = DataAccessLayer() - fm = FAISSIndexManager(da) - try: - fm.initialize_index(force_rebuild=False) - except Exception as e: # pragma: no cover - environment dependent - pytest.skip(f"Skipping alignment determinism test: index init failed: {repr(e)}") - return SkillAlignmentService(data_access=da, faiss_manager=fm) - - -@pytest.mark.alignment -def test_alignment_is_stable_across_repeated_runs(alignment_service): - """The alignment stage must be reproducible. - - Retrieval uses an exact inner-product index over deterministic - sentence-transformer embeddings, so repeated alignment of an identical raw - skill list must produce an identical frame, including similarity scores. - """ - runs = [ - alignment_service.align_skills_to_taxonomy(list(RAW_SKILLS), document_id="doc-1") - for _ in range(3) - ] - - first = runs[0].reset_index(drop=True) - for i, other in enumerate(runs[1:], start=2): - pd.testing.assert_frame_equal( - first, other.reset_index(drop=True), check_exact=True, - obj=f"alignment run 1 vs run {i}", - ) - - -@pytest.mark.alignment -def test_alignment_output_space_is_closed(alignment_service): - """Aligned output must be drawn from the taxonomy, never from free text. - - This is the mechanism that makes aligned results comparable across - backends: whatever a model generates, the reported canonical skill is - always an entry that already exists in the taxonomy. - """ - aligned = alignment_service.align_skills_to_taxonomy(list(RAW_SKILLS), document_id="doc-2") - - if aligned.empty: # pragma: no cover - environment dependent - pytest.skip("Alignment returned no rows; taxonomy index may be unavailable") - - metadata = alignment_service.faiss_manager.get_metadata() - vocabulary = set(metadata["skill"].astype(str)) - - unknown = set(aligned["Taxonomy Skill"]) - vocabulary - assert not unknown, f"Aligned output contains entries absent from the taxonomy: {unknown}" - - -@pytest.mark.alignment -def test_identical_raw_skills_map_to_identical_taxonomy_entries(alignment_service): - """The same raw string must always resolve to the same taxonomy entry. - - Alignment is a pure function of the raw string, so a phrase repeated within - one document, or emitted by two different backends, cannot produce two - different canonical skills or two different similarity scores. - """ - duplicated = ["statistical analysis", "Python programming", "statistical analysis"] - aligned = alignment_service.align_skills_to_taxonomy(duplicated, document_id="doc-3") - - if aligned.empty: # pragma: no cover - environment dependent - pytest.skip("Alignment returned no rows; taxonomy index may be unavailable") - - for raw_skill, group in aligned.groupby("Raw Skill"): - assert group["Taxonomy Skill"].nunique() == 1, ( - f"Raw skill {raw_skill!r} resolved to multiple taxonomy entries: " - f"{set(group['Taxonomy Skill'])}" - ) - assert group["Correlation Coefficient"].nunique() == 1, ( - f"Raw skill {raw_skill!r} produced inconsistent similarity scores: " - f"{set(group['Correlation Coefficient'])}" - )