From b72f9e82a3557cb76b1d05b34e7521dd55d275d5 Mon Sep 17 00:00:00 2001 From: Eike Haller <58111764+eksrha@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:33:50 +0100 Subject: [PATCH] feat(router): publish model capabilities on /v1/models and /v1/models/{id} Clients can see per model whether a prompt or hotwords have any effect, whether word timestamps and confidence are returned, and the longest audio and largest upload accepted, instead of guessing from the model name. The values come from the router configuration (defaults match the bundled servers, override with _CAPABILITIES or router.capabilities in the chart); what a backend claims about itself is never passed through. The field is additive. --- AGENTS.md | 2 +- README.md | 24 ++++- deploy/helm/templates/router/deployment.yaml | 6 ++ deploy/helm/values.yaml | 7 ++ internal/config/capabilities.go | 81 +++++++++++++++++ internal/config/config.go | 22 +++++ internal/config/config_test.go | 57 ++++++++++++ internal/proxy/proxy.go | 52 +++++++++-- internal/proxy/proxy_test.go | 92 +++++++++++++++++++- 9 files changed, 333 insertions(+), 10 deletions(-) create mode 100644 internal/config/capabilities.go create mode 100644 internal/config/config_test.go diff --git a/AGENTS.md b/AGENTS.md index ab83b5c..71a62a3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ This file provides context for AI coding agents working in this repository. | Whisper ASR | faster-whisper (Python) | large-v3-turbo | `/v1/audio/transcriptions`, `/v1/audio/translations` | `WHISPER_ENABLED`, `WHISPER_URL` | | Parakeet ASR | onnx-asr + ONNX Runtime (Python) | parakeet-tdt-0.6b-v3 (int8) | `/v1/audio/transcriptions` when `model` names it | `PARAKEET_ENABLED`, `PARAKEET_URL` | -`/v1/models` returns a merged list from all enabled backends (with static fallback). +`/v1/models` (and `/v1/models/{id}`) returns a merged list from all enabled backends (with static fallback); each entry carries the configured `capabilities` of its backend (`internal/config/capabilities.go`, override with `_CAPABILITIES`). --- diff --git a/README.md b/README.md index 85a41ae..cad260b 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ batch tokens) is in [docs/tuning.md](docs/tuning.md); all chart options are comm | `POST /v1/rerank`, `POST /rerank` | reranker | `/v1/rerank` is rewritten to TEI's `/rerank` | | `POST /v1/audio/transcriptions` | whisper, parakeet | multipart; `json`, `text`, `verbose_json`, `srt`, `vtt`. Goes to parakeet only when it is enabled and `model` is `parakeet-tdt-0.6b-v3` | | `POST /v1/audio/translations` | whisper | exposed, but `large-v3-turbo` is not trained for translation | -| `GET /v1/models` | all enabled | merged list incl. `limits` of embedding and reranker | +| `GET /v1/models`, `GET /v1/models/{id}` | all enabled | merged list incl. `limits` of embedding and reranker and the [`capabilities`](#model-capabilities) of every model | | `GET /healthz`, `GET /readyz` | router | per-backend status, always HTTP 200 (`ok` or `degraded`) | | `GET :9090/metrics` | router | Prometheus | @@ -126,6 +126,28 @@ curl http://localhost:8080/v1/audio/transcriptions -F model=whisper-large-v3-tur A disabled backend answers `503 backend_disabled`; an unreachable one `502 upstream_error`; one at its concurrency bound `503 backend_overloaded` with `Retry-After`. +## Model capabilities + +Every entry of `GET /v1/models` (and `GET /v1/models/{id}`) carries an additive `capabilities` object so that a client can adapt to the model behind the router instead of guessing from its name. The values come from the router's configuration, never from what a backend claims about itself. Defaults match the bundled servers; replace them per backend with `_CAPABILITIES` (JSON, `NAME` one of `EMBEDDING`, `RERANKER`, `WHISPER`, `PARAKEET`) or `router.capabilities.` in the chart. A malformed value stops the router at start. + +```json +{ + "id": "parakeet-tdt-0.6b-v3", + "object": "model", + "capabilities": { + "tasks": ["transcribe"], + "audio": { + "prompt": false, "hotwords": false, + "word_times": true, "segment_times": true, "confidence": true, + "auto_language": true, "force_language": false, + "max_seconds": 120, "max_bytes": 26214400 + } + } +} +``` + +`audio.prompt` and `audio.hotwords` tell a client whether a `prompt` or `hotwords` form field has any effect (a backend may accept and ignore it), `max_seconds` and `max_bytes` are per request (0 or absent: not stated), `languages` lists ISO 639-1 codes when the model has a fixed set. Unknown fields must be ignored by clients. + ## Components and images | Component | Image(s) in `ghcr.io/layer87-labs/` | Purpose | diff --git a/deploy/helm/templates/router/deployment.yaml b/deploy/helm/templates/router/deployment.yaml index 368e056..5dfc534 100644 --- a/deploy/helm/templates/router/deployment.yaml +++ b/deploy/helm/templates/router/deployment.yaml @@ -77,6 +77,12 @@ spec: - name: PARAKEET_URL value: "http://{{ include "inference-stack.fullname" . }}-parakeet:{{ .Values.parakeet.service.port }}" {{- end }} + {{- range $name, $json := .Values.router.capabilities }} + {{- if $json }} + - name: {{ upper $name }}_CAPABILITIES + value: {{ $json | trim | quote }} + {{- end }} + {{- end }} {{- with .Values.router.extraEnv }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 66fb05d..90ad010 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -49,6 +49,13 @@ router: healthCheckInterval: "15s" + # Capabilities published on /v1/models and /v1/models/{id} per backend. The + # router has defaults that match the bundled servers; set a backend's key + # (embedding, reranker, whisper, parakeet) to a JSON document to replace + # them, for example when you changed a server limit: + # whisper: '{"tasks":["transcribe"],"audio":{"prompt":true,"hotwords":true,"word_times":true,"segment_times":true,"max_bytes":52428800}}' + capabilities: {} + # Extra environment variables for the router, e.g. to tune per-backend # timeouts (WHISPER_TIMEOUT, ROUTER_WRITE_TIMEOUT, ...). List of EnvVar. extraEnv: [] diff --git a/internal/config/capabilities.go b/internal/config/capabilities.go new file mode 100644 index 0000000..e36f220 --- /dev/null +++ b/internal/config/capabilities.go @@ -0,0 +1,81 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" +) + +// Capabilities describe what the model(s) behind a backend can do. They are +// published as the "capabilities" field of /v1/models entries, so that a +// client can adapt (send a prompt or not, ask for word timestamps, cut audio +// to the longest accepted length) instead of guessing from the model name. +// The field is additive: clients that do not know it ignore it. +type Capabilities struct { + Tasks []string `json:"tasks,omitempty"` // "embed", "rerank", "transcribe" + Languages []string `json:"languages,omitempty"` // ISO 639-1; empty = not stated + Audio *AudioCapabilities `json:"audio,omitempty"` +} + +// AudioCapabilities are the abilities of a transcription backend. +type AudioCapabilities struct { + Prompt bool `json:"prompt"` // accepts a context prompt (`prompt` form field) + PromptMax int `json:"prompt_max,omitempty"` // characters; 0 = not stated + Hotwords bool `json:"hotwords"` // accepts a `hotwords` form field + HotwordsMax int `json:"hotwords_max,omitempty"` // characters; 0 = not stated + WordTimes bool `json:"word_times"` // word timestamps in verbose_json + SegmentTimes bool `json:"segment_times"` // segment timestamps in verbose_json + Confidence bool `json:"confidence"` // per-segment or per-word confidence + AutoLanguage bool `json:"auto_language"` // detects the language itself + ForceLang bool `json:"force_language"` // honours the `language` field + Diarization bool `json:"diarization"` // separates speakers + Streaming bool `json:"streaming"` // takes audio while it is spoken + MaxSeconds int `json:"max_seconds,omitempty"` // longest audio per request; 0 = not stated + MaxBytes int `json:"max_bytes,omitempty"` // largest upload per request; 0 = not stated + Formats []string `json:"formats,omitempty"` // accepted containers; empty = not stated +} + +func (c *Capabilities) validate() error { + if a := c.Audio; a != nil && (a.PromptMax < 0 || a.HotwordsMax < 0 || a.MaxSeconds < 0 || a.MaxBytes < 0) { + return fmt.Errorf("negative limit") + } + return nil +} + +// capabilitiesFromEnv returns the JSON in the environment variable key, +// or def when it is unset. A malformed value is an error: a typo must not +// silently publish the defaults. +func capabilitiesFromEnv(key string, def *Capabilities) (*Capabilities, error) { + v := os.Getenv(key) + if v == "" { + return def, nil + } + var c Capabilities + if err := json.Unmarshal([]byte(v), &c); err != nil { + return nil, fmt.Errorf("%s: invalid capabilities JSON: %w", key, err) + } + if err := c.validate(); err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + return &c, nil +} + +// Defaults for the bundled backends, matching what the servers in deploy/ do. +func embeddingCapabilities() *Capabilities { return &Capabilities{Tasks: []string{"embed"}} } +func rerankerCapabilities() *Capabilities { return &Capabilities{Tasks: []string{"rerank"}} } + +func whisperCapabilities() *Capabilities { + return &Capabilities{Tasks: []string{"transcribe"}, Audio: &AudioCapabilities{ + Prompt: true, PromptMax: 2000, Hotwords: true, HotwordsMax: 2000, + WordTimes: true, SegmentTimes: true, Confidence: true, AutoLanguage: true, ForceLang: true, + MaxBytes: 100 << 20, + }} +} + +func parakeetCapabilities() *Capabilities { + return &Capabilities{Tasks: []string{"transcribe"}, Audio: &AudioCapabilities{ + // The prompt field is accepted and ignored: the model has no prompt input. + WordTimes: true, SegmentTimes: true, Confidence: true, AutoLanguage: true, + MaxSeconds: 120, MaxBytes: 25 << 20, + }} +} diff --git a/internal/config/config.go b/internal/config/config.go index 2b76b13..52a071d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,6 +27,11 @@ type Backend struct { // under load. 0 or negative disables the limit (unbounded passthrough, // the previous behavior). MaxConcurrent int + + // Capabilities are published on /v1/models for the models of this + // backend (nil: not published). Defaults for the bundled backends can be + // replaced with _CAPABILITIES (JSON). + Capabilities *Capabilities } // Config holds all runtime configuration for the inference router. @@ -113,6 +118,23 @@ func Load() (*Config, error) { }, } + for _, c := range []struct { + b *Backend + key string + def *Capabilities + }{ + {&cfg.Embedding, "EMBEDDING_CAPABILITIES", embeddingCapabilities()}, + {&cfg.Reranker, "RERANKER_CAPABILITIES", rerankerCapabilities()}, + {&cfg.Whisper, "WHISPER_CAPABILITIES", whisperCapabilities()}, + {&cfg.Parakeet, "PARAKEET_CAPABILITIES", parakeetCapabilities()}, + } { + caps, err := capabilitiesFromEnv(c.key, c.def) + if err != nil { + return nil, err + } + c.b.Capabilities = caps + } + if err := cfg.validate(); err != nil { return nil, err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..634e49c --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,57 @@ +package config + +import ( + "reflect" + "strings" + "testing" +) + +func setEnabled(t *testing.T) { + t.Helper() + t.Setenv("WHISPER_ENABLED", "true") + t.Setenv("WHISPER_URL", "http://whisper:8000") + t.Setenv("PARAKEET_ENABLED", "true") + t.Setenv("PARAKEET_URL", "http://parakeet:8000") +} + +func TestLoad_CapabilityDefaults(t *testing.T) { + setEnabled(t) + cfg, err := Load() + if err != nil { + t.Fatal(err) + } + w, p := cfg.Whisper.Capabilities.Audio, cfg.Parakeet.Capabilities.Audio + if !w.Prompt || !w.Hotwords || !w.WordTimes || !w.ForceLang || w.MaxBytes != 100<<20 { + t.Errorf("whisper: %+v", w) + } + // The second backend differs: no prompt, no hotwords, a short audio limit. + if p.Prompt || p.Hotwords || !p.WordTimes || p.ForceLang || p.MaxSeconds != 120 || p.MaxBytes != 25<<20 { + t.Errorf("parakeet: %+v", p) + } + if !reflect.DeepEqual(cfg.Embedding.Capabilities.Tasks, []string{"embed"}) || !reflect.DeepEqual(cfg.Reranker.Capabilities.Tasks, []string{"rerank"}) { + t.Errorf("tasks: %+v %+v", cfg.Embedding.Capabilities, cfg.Reranker.Capabilities) + } +} + +func TestLoad_CapabilityOverride(t *testing.T) { + setEnabled(t) + t.Setenv("WHISPER_CAPABILITIES", `{"tasks":["transcribe"],"languages":["de","en"],"audio":{"prompt":false,"hotwords":true,"max_seconds":300}}`) + cfg, err := Load() + if err != nil { + t.Fatal(err) + } + c := cfg.Whisper.Capabilities + if !reflect.DeepEqual(c.Languages, []string{"de", "en"}) || c.Audio.Prompt || !c.Audio.Hotwords || c.Audio.MaxSeconds != 300 { + t.Errorf("%+v %+v", c, c.Audio) + } +} + +func TestLoad_BadCapabilitiesFail(t *testing.T) { + setEnabled(t) + for name, v := range map[string]string{"not json": `{"audio":`, "negative": `{"audio":{"max_bytes":-1}}`} { + t.Setenv("PARAKEET_CAPABILITIES", v) + if _, err := Load(); err == nil || !strings.Contains(err.Error(), "PARAKEET_CAPABILITIES") { + t.Errorf("%s: got %v", name, err) + } + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 6b91586..0b76a76 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -122,6 +122,9 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { case path == "/v1/models" || path == "/v1/models/": r.handleModels(w, req) + case strings.HasPrefix(path, "/v1/models/") && req.Method == http.MethodGet: + r.handleModel(w, strings.TrimPrefix(path, "/v1/models/")) + case strings.HasPrefix(path, "/v1/embeddings") || path == "/embed" || path == "/embed_sparse": if strings.HasPrefix(path, "/v1/embeddings") { r.stripAdvertisedModel(req, r.cfg.Embedding) @@ -239,6 +242,9 @@ type modelObj struct { Created int64 `json:"created"` OwnedBy string `json:"owned_by"` Limits *modelLimits `json:"limits,omitempty"` + // Capabilities say what the model can do (prompt, word timestamps, longest + // audio, ...), from the backend's configuration. Additive field. + Capabilities *config.Capabilities `json:"capabilities,omitempty"` } // modelLimits are the request-size limits of the running backend, read from @@ -392,7 +398,31 @@ type modelsResp struct { } // handleModels returns a merged model list from all enabled backends. -func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { +func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(modelsResp{Object: "list", Data: r.collectModels()}) +} + +// handleModel serves GET /v1/models/{id}: one entry of the merged list, in the +// OpenAI shape, or a 404 error object. +func (r *Router) handleModel(w http.ResponseWriter, id string) { + for _, m := range r.collectModels() { + if m.ID == id { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(m) + return + } + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{ + "message": "model not found", "type": "invalid_request_error", "code": "model_not_found", + }}) +} + +// collectModels merges the model lists of all enabled backends and attaches +// each backend's limits and capabilities. +func (r *Router) collectModels() []modelObj { var models []modelObj client := &http.Client{Timeout: 5 * time.Second} @@ -401,20 +431,20 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { if err != nil { r.log.Warn("failed to fetch models from backend", zap.String("backend", b.Name), zap.Error(err)) - models = append(models, r.staticModels(b)...) + models = append(models, r.withCapabilities(b, r.staticModels(b))...) continue } body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<16)) _ = resp.Body.Close() if err != nil || resp.StatusCode != http.StatusOK { - models = append(models, r.staticModels(b)...) + models = append(models, r.withCapabilities(b, r.staticModels(b))...) continue } var upstreamResp modelsResp if err := json.Unmarshal(body, &upstreamResp); err != nil { - models = append(models, r.staticModels(b)...) + models = append(models, r.withCapabilities(b, r.staticModels(b))...) continue } @@ -427,11 +457,19 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { valid[i].Limits = limits } } - models = append(models, valid...) + models = append(models, r.withCapabilities(b, valid)...) } + return models +} - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(modelsResp{Object: "list", Data: models}) +// withCapabilities sets the backend's configured capabilities on its models. +func (r *Router) withCapabilities(b config.Backend, ms []modelObj) []modelObj { + // Always set, also to nil: what a backend claims about itself on + // /v1/models is never passed through, the configuration decides. + for i := range ms { + ms[i].Capabilities = b.Capabilities + } + return ms } // validateModels filters out entries with an unusable "id". diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 14c8787..a2f9635 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -721,7 +721,7 @@ func asrBackends(t *testing.T) (cfg *config.Config, got map[string]string) { "/v1/audio/transcriptions": func(w http.ResponseWriter, r *http.Request) { b, _ := io.ReadAll(r.Body) got[name] = string(b) - jsonOK(`{"text":"` + name + `"}`)(w, r) + jsonOK(`{"text":"`+name+`"}`)(w, r) }, "/v1/audio/translations": jsonOK(`{"text":"` + name + `"}`), }) @@ -821,3 +821,93 @@ func TestRouting_TranslationsStayOnWhisper(t *testing.T) { t.Errorf("got %s, want whisper", w.Body.String()) } } + +// ── Capabilities ────────────────────────────────────────────────────────────── + +func capabilityConfig(whisperURL, embedURL string) *config.Config { + return buildConfig( + config.Backend{Name: "embedding", BaseURL: embedURL, Enabled: true, Timeout: 5 * time.Second, Models: []string{"BAAI/bge-m3"}, + Capabilities: &config.Capabilities{Tasks: []string{"embed"}}}, + config.Backend{Name: "reranker"}, + config.Backend{Name: "whisper", BaseURL: whisperURL, Enabled: true, Timeout: 5 * time.Second, + Capabilities: &config.Capabilities{Tasks: []string{"transcribe"}, Audio: &config.AudioCapabilities{ + Prompt: true, Hotwords: true, WordTimes: true, MaxSeconds: 900}}}, + ) +} + +func TestModels_CarryConfiguredCapabilities(t *testing.T) { + whisper := mockBackend(t, map[string]http.HandlerFunc{ + // The backend claims capabilities of its own: they must not pass through. + "/v1/models": jsonOK(`{"object":"list","data":[{"id":"whisper-large-v3","object":"model","created":1,"owned_by":"x","capabilities":{"tasks":["lie"]}}]}`), + }) + embed := mockBackend(t, map[string]http.HandlerFunc{"/v1/models": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) }}) + router := buildRouter(t, capabilityConfig(whisper.URL, embed.URL)) + + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + var resp struct { + Data []struct { + ID string `json:"id"` + Capabilities *struct { + Tasks []string `json:"tasks"` + Audio *struct { + Prompt bool `json:"prompt"` + Hotwords bool `json:"hotwords"` + WordTimes bool `json:"word_times"` + MaxSeconds int `json:"max_seconds"` + } `json:"audio"` + } `json:"capabilities"` + } `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil || len(resp.Data) != 2 { + t.Fatalf("%v\n%s", err, w.Body.String()) + } + byID := map[string]int{} + for i, m := range resp.Data { + byID[m.ID] = i + } + wm := resp.Data[byID["whisper-large-v3"]] + if wm.Capabilities == nil || !reflect.DeepEqual(wm.Capabilities.Tasks, []string{"transcribe"}) || wm.Capabilities.Audio == nil || + !wm.Capabilities.Audio.Prompt || !wm.Capabilities.Audio.Hotwords || wm.Capabilities.Audio.MaxSeconds != 900 { + t.Fatalf("whisper: %+v", wm.Capabilities) + } + // The embedding backend failed upstream: static fallback still carries them. + em := resp.Data[byID["BAAI/bge-m3"]] + if em.Capabilities == nil || !reflect.DeepEqual(em.Capabilities.Tasks, []string{"embed"}) || em.Capabilities.Audio != nil { + t.Fatalf("embedding: %+v", em.Capabilities) + } +} + +func TestModels_NoCapabilitiesWhenNotConfigured(t *testing.T) { + whisper := mockBackend(t, map[string]http.HandlerFunc{"/v1/models": jsonOK(modelsJSON("whisper-large-v3"))}) + cfg := buildConfig(config.Backend{}, config.Backend{}, config.Backend{Name: "whisper", BaseURL: whisper.URL, Enabled: true, Timeout: 5 * time.Second}) + w := httptest.NewRecorder() + buildRouter(t, cfg).ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + if strings.Contains(w.Body.String(), "capabilities") { + t.Fatalf("field present without configuration: %s", w.Body.String()) + } +} + +func TestModelByID(t *testing.T) { + whisper := mockBackend(t, map[string]http.HandlerFunc{"/v1/models": jsonOK(modelsJSON("whisper-large-v3"))}) + embed := mockBackend(t, map[string]http.HandlerFunc{"/v1/models": jsonOK(modelsJSON("BAAI/bge-m3"))}) + router := buildRouter(t, capabilityConfig(whisper.URL, embed.URL)) + + for _, id := range []string{"whisper-large-v3", "BAAI/bge-m3"} { // ids may contain a slash + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/models/"+id, nil)) + var m struct { + ID string `json:"id"` + Object string `json:"object"` + Capabilities json.RawMessage `json:"capabilities"` + } + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &m) != nil || m.ID != id || m.Object != "model" || len(m.Capabilities) == 0 { + t.Fatalf("%s: %d %s", id, w.Code, w.Body.String()) + } + } + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/models/nope", nil)) + if w.Code != http.StatusNotFound || !strings.Contains(w.Body.String(), "model_not_found") { + t.Fatalf("unknown id: %d %s", w.Code, w.Body.String()) + } +}