diff --git a/AGENTS.md b/AGENTS.md index ab83b5c..71a62a3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ This file provides context for AI coding agents working in this repository. | Whisper ASR | faster-whisper (Python) | large-v3-turbo | `/v1/audio/transcriptions`, `/v1/audio/translations` | `WHISPER_ENABLED`, `WHISPER_URL` | | Parakeet ASR | onnx-asr + ONNX Runtime (Python) | parakeet-tdt-0.6b-v3 (int8) | `/v1/audio/transcriptions` when `model` names it | `PARAKEET_ENABLED`, `PARAKEET_URL` | -`/v1/models` returns a merged list from all enabled backends (with static fallback). +`/v1/models` (and `/v1/models/{id}`) returns a merged list from all enabled backends (with static fallback); each entry carries the configured `capabilities` of its backend (`internal/config/capabilities.go`, override with `_CAPABILITIES`). --- diff --git a/README.md b/README.md index 85a41ae..cad260b 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ batch tokens) is in [docs/tuning.md](docs/tuning.md); all chart options are comm | `POST /v1/rerank`, `POST /rerank` | reranker | `/v1/rerank` is rewritten to TEI's `/rerank` | | `POST /v1/audio/transcriptions` | whisper, parakeet | multipart; `json`, `text`, `verbose_json`, `srt`, `vtt`. Goes to parakeet only when it is enabled and `model` is `parakeet-tdt-0.6b-v3` | | `POST /v1/audio/translations` | whisper | exposed, but `large-v3-turbo` is not trained for translation | -| `GET /v1/models` | all enabled | merged list incl. `limits` of embedding and reranker | +| `GET /v1/models`, `GET /v1/models/{id}` | all enabled | merged list incl. `limits` of embedding and reranker and the [`capabilities`](#model-capabilities) of every model | | `GET /healthz`, `GET /readyz` | router | per-backend status, always HTTP 200 (`ok` or `degraded`) | | `GET :9090/metrics` | router | Prometheus | @@ -126,6 +126,28 @@ curl http://localhost:8080/v1/audio/transcriptions -F model=whisper-large-v3-tur A disabled backend answers `503 backend_disabled`; an unreachable one `502 upstream_error`; one at its concurrency bound `503 backend_overloaded` with `Retry-After`. +## Model capabilities + +Every entry of `GET /v1/models` (and `GET /v1/models/{id}`) carries an additive `capabilities` object so that a client can adapt to the model behind the router instead of guessing from its name. The values come from the router's configuration, never from what a backend claims about itself. Defaults match the bundled servers; replace them per backend with `_CAPABILITIES` (JSON, `NAME` one of `EMBEDDING`, `RERANKER`, `WHISPER`, `PARAKEET`) or `router.capabilities.` in the chart. A malformed value stops the router at start. + +```json +{ + "id": "parakeet-tdt-0.6b-v3", + "object": "model", + "capabilities": { + "tasks": ["transcribe"], + "audio": { + "prompt": false, "hotwords": false, + "word_times": true, "segment_times": true, "confidence": true, + "auto_language": true, "force_language": false, + "max_seconds": 120, "max_bytes": 26214400 + } + } +} +``` + +`audio.prompt` and `audio.hotwords` tell a client whether a `prompt` or `hotwords` form field has any effect (a backend may accept and ignore it), `max_seconds` and `max_bytes` are per request (0 or absent: not stated), `languages` lists ISO 639-1 codes when the model has a fixed set. Unknown fields must be ignored by clients. + ## Components and images | Component | Image(s) in `ghcr.io/layer87-labs/` | Purpose | diff --git a/deploy/helm/templates/router/deployment.yaml b/deploy/helm/templates/router/deployment.yaml index 368e056..5dfc534 100644 --- a/deploy/helm/templates/router/deployment.yaml +++ b/deploy/helm/templates/router/deployment.yaml @@ -77,6 +77,12 @@ spec: - name: PARAKEET_URL value: "http://{{ include "inference-stack.fullname" . }}-parakeet:{{ .Values.parakeet.service.port }}" {{- end }} + {{- range $name, $json := .Values.router.capabilities }} + {{- if $json }} + - name: {{ upper $name }}_CAPABILITIES + value: {{ $json | trim | quote }} + {{- end }} + {{- end }} {{- with .Values.router.extraEnv }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 66fb05d..90ad010 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -49,6 +49,13 @@ router: healthCheckInterval: "15s" + # Capabilities published on /v1/models and /v1/models/{id} per backend. The + # router has defaults that match the bundled servers; set a backend's key + # (embedding, reranker, whisper, parakeet) to a JSON document to replace + # them, for example when you changed a server limit: + # whisper: '{"tasks":["transcribe"],"audio":{"prompt":true,"hotwords":true,"word_times":true,"segment_times":true,"max_bytes":52428800}}' + capabilities: {} + # Extra environment variables for the router, e.g. to tune per-backend # timeouts (WHISPER_TIMEOUT, ROUTER_WRITE_TIMEOUT, ...). List of EnvVar. extraEnv: [] diff --git a/internal/config/capabilities.go b/internal/config/capabilities.go new file mode 100644 index 0000000..e36f220 --- /dev/null +++ b/internal/config/capabilities.go @@ -0,0 +1,81 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" +) + +// Capabilities describe what the model(s) behind a backend can do. They are +// published as the "capabilities" field of /v1/models entries, so that a +// client can adapt (send a prompt or not, ask for word timestamps, cut audio +// to the longest accepted length) instead of guessing from the model name. +// The field is additive: clients that do not know it ignore it. +type Capabilities struct { + Tasks []string `json:"tasks,omitempty"` // "embed", "rerank", "transcribe" + Languages []string `json:"languages,omitempty"` // ISO 639-1; empty = not stated + Audio *AudioCapabilities `json:"audio,omitempty"` +} + +// AudioCapabilities are the abilities of a transcription backend. +type AudioCapabilities struct { + Prompt bool `json:"prompt"` // accepts a context prompt (`prompt` form field) + PromptMax int `json:"prompt_max,omitempty"` // characters; 0 = not stated + Hotwords bool `json:"hotwords"` // accepts a `hotwords` form field + HotwordsMax int `json:"hotwords_max,omitempty"` // characters; 0 = not stated + WordTimes bool `json:"word_times"` // word timestamps in verbose_json + SegmentTimes bool `json:"segment_times"` // segment timestamps in verbose_json + Confidence bool `json:"confidence"` // per-segment or per-word confidence + AutoLanguage bool `json:"auto_language"` // detects the language itself + ForceLang bool `json:"force_language"` // honours the `language` field + Diarization bool `json:"diarization"` // separates speakers + Streaming bool `json:"streaming"` // takes audio while it is spoken + MaxSeconds int `json:"max_seconds,omitempty"` // longest audio per request; 0 = not stated + MaxBytes int `json:"max_bytes,omitempty"` // largest upload per request; 0 = not stated + Formats []string `json:"formats,omitempty"` // accepted containers; empty = not stated +} + +func (c *Capabilities) validate() error { + if a := c.Audio; a != nil && (a.PromptMax < 0 || a.HotwordsMax < 0 || a.MaxSeconds < 0 || a.MaxBytes < 0) { + return fmt.Errorf("negative limit") + } + return nil +} + +// capabilitiesFromEnv returns the JSON in the environment variable key, +// or def when it is unset. A malformed value is an error: a typo must not +// silently publish the defaults. +func capabilitiesFromEnv(key string, def *Capabilities) (*Capabilities, error) { + v := os.Getenv(key) + if v == "" { + return def, nil + } + var c Capabilities + if err := json.Unmarshal([]byte(v), &c); err != nil { + return nil, fmt.Errorf("%s: invalid capabilities JSON: %w", key, err) + } + if err := c.validate(); err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + return &c, nil +} + +// Defaults for the bundled backends, matching what the servers in deploy/ do. +func embeddingCapabilities() *Capabilities { return &Capabilities{Tasks: []string{"embed"}} } +func rerankerCapabilities() *Capabilities { return &Capabilities{Tasks: []string{"rerank"}} } + +func whisperCapabilities() *Capabilities { + return &Capabilities{Tasks: []string{"transcribe"}, Audio: &AudioCapabilities{ + Prompt: true, PromptMax: 2000, Hotwords: true, HotwordsMax: 2000, + WordTimes: true, SegmentTimes: true, Confidence: true, AutoLanguage: true, ForceLang: true, + MaxBytes: 100 << 20, + }} +} + +func parakeetCapabilities() *Capabilities { + return &Capabilities{Tasks: []string{"transcribe"}, Audio: &AudioCapabilities{ + // The prompt field is accepted and ignored: the model has no prompt input. + WordTimes: true, SegmentTimes: true, Confidence: true, AutoLanguage: true, + MaxSeconds: 120, MaxBytes: 25 << 20, + }} +} diff --git a/internal/config/config.go b/internal/config/config.go index 2b76b13..52a071d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -27,6 +27,11 @@ type Backend struct { // under load. 0 or negative disables the limit (unbounded passthrough, // the previous behavior). MaxConcurrent int + + // Capabilities are published on /v1/models for the models of this + // backend (nil: not published). Defaults for the bundled backends can be + // replaced with _CAPABILITIES (JSON). + Capabilities *Capabilities } // Config holds all runtime configuration for the inference router. @@ -113,6 +118,23 @@ func Load() (*Config, error) { }, } + for _, c := range []struct { + b *Backend + key string + def *Capabilities + }{ + {&cfg.Embedding, "EMBEDDING_CAPABILITIES", embeddingCapabilities()}, + {&cfg.Reranker, "RERANKER_CAPABILITIES", rerankerCapabilities()}, + {&cfg.Whisper, "WHISPER_CAPABILITIES", whisperCapabilities()}, + {&cfg.Parakeet, "PARAKEET_CAPABILITIES", parakeetCapabilities()}, + } { + caps, err := capabilitiesFromEnv(c.key, c.def) + if err != nil { + return nil, err + } + c.b.Capabilities = caps + } + if err := cfg.validate(); err != nil { return nil, err } diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..634e49c --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,57 @@ +package config + +import ( + "reflect" + "strings" + "testing" +) + +func setEnabled(t *testing.T) { + t.Helper() + t.Setenv("WHISPER_ENABLED", "true") + t.Setenv("WHISPER_URL", "http://whisper:8000") + t.Setenv("PARAKEET_ENABLED", "true") + t.Setenv("PARAKEET_URL", "http://parakeet:8000") +} + +func TestLoad_CapabilityDefaults(t *testing.T) { + setEnabled(t) + cfg, err := Load() + if err != nil { + t.Fatal(err) + } + w, p := cfg.Whisper.Capabilities.Audio, cfg.Parakeet.Capabilities.Audio + if !w.Prompt || !w.Hotwords || !w.WordTimes || !w.ForceLang || w.MaxBytes != 100<<20 { + t.Errorf("whisper: %+v", w) + } + // The second backend differs: no prompt, no hotwords, a short audio limit. + if p.Prompt || p.Hotwords || !p.WordTimes || p.ForceLang || p.MaxSeconds != 120 || p.MaxBytes != 25<<20 { + t.Errorf("parakeet: %+v", p) + } + if !reflect.DeepEqual(cfg.Embedding.Capabilities.Tasks, []string{"embed"}) || !reflect.DeepEqual(cfg.Reranker.Capabilities.Tasks, []string{"rerank"}) { + t.Errorf("tasks: %+v %+v", cfg.Embedding.Capabilities, cfg.Reranker.Capabilities) + } +} + +func TestLoad_CapabilityOverride(t *testing.T) { + setEnabled(t) + t.Setenv("WHISPER_CAPABILITIES", `{"tasks":["transcribe"],"languages":["de","en"],"audio":{"prompt":false,"hotwords":true,"max_seconds":300}}`) + cfg, err := Load() + if err != nil { + t.Fatal(err) + } + c := cfg.Whisper.Capabilities + if !reflect.DeepEqual(c.Languages, []string{"de", "en"}) || c.Audio.Prompt || !c.Audio.Hotwords || c.Audio.MaxSeconds != 300 { + t.Errorf("%+v %+v", c, c.Audio) + } +} + +func TestLoad_BadCapabilitiesFail(t *testing.T) { + setEnabled(t) + for name, v := range map[string]string{"not json": `{"audio":`, "negative": `{"audio":{"max_bytes":-1}}`} { + t.Setenv("PARAKEET_CAPABILITIES", v) + if _, err := Load(); err == nil || !strings.Contains(err.Error(), "PARAKEET_CAPABILITIES") { + t.Errorf("%s: got %v", name, err) + } + } +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index 6b91586..0b76a76 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -122,6 +122,9 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { case path == "/v1/models" || path == "/v1/models/": r.handleModels(w, req) + case strings.HasPrefix(path, "/v1/models/") && req.Method == http.MethodGet: + r.handleModel(w, strings.TrimPrefix(path, "/v1/models/")) + case strings.HasPrefix(path, "/v1/embeddings") || path == "/embed" || path == "/embed_sparse": if strings.HasPrefix(path, "/v1/embeddings") { r.stripAdvertisedModel(req, r.cfg.Embedding) @@ -239,6 +242,9 @@ type modelObj struct { Created int64 `json:"created"` OwnedBy string `json:"owned_by"` Limits *modelLimits `json:"limits,omitempty"` + // Capabilities say what the model can do (prompt, word timestamps, longest + // audio, ...), from the backend's configuration. Additive field. + Capabilities *config.Capabilities `json:"capabilities,omitempty"` } // modelLimits are the request-size limits of the running backend, read from @@ -392,7 +398,31 @@ type modelsResp struct { } // handleModels returns a merged model list from all enabled backends. -func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { +func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(modelsResp{Object: "list", Data: r.collectModels()}) +} + +// handleModel serves GET /v1/models/{id}: one entry of the merged list, in the +// OpenAI shape, or a 404 error object. +func (r *Router) handleModel(w http.ResponseWriter, id string) { + for _, m := range r.collectModels() { + if m.ID == id { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(m) + return + } + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{ + "message": "model not found", "type": "invalid_request_error", "code": "model_not_found", + }}) +} + +// collectModels merges the model lists of all enabled backends and attaches +// each backend's limits and capabilities. +func (r *Router) collectModels() []modelObj { var models []modelObj client := &http.Client{Timeout: 5 * time.Second} @@ -401,20 +431,20 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { if err != nil { r.log.Warn("failed to fetch models from backend", zap.String("backend", b.Name), zap.Error(err)) - models = append(models, r.staticModels(b)...) + models = append(models, r.withCapabilities(b, r.staticModels(b))...) continue } body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<16)) _ = resp.Body.Close() if err != nil || resp.StatusCode != http.StatusOK { - models = append(models, r.staticModels(b)...) + models = append(models, r.withCapabilities(b, r.staticModels(b))...) continue } var upstreamResp modelsResp if err := json.Unmarshal(body, &upstreamResp); err != nil { - models = append(models, r.staticModels(b)...) + models = append(models, r.withCapabilities(b, r.staticModels(b))...) continue } @@ -427,11 +457,19 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { valid[i].Limits = limits } } - models = append(models, valid...) + models = append(models, r.withCapabilities(b, valid)...) } + return models +} - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(modelsResp{Object: "list", Data: models}) +// withCapabilities sets the backend's configured capabilities on its models. +func (r *Router) withCapabilities(b config.Backend, ms []modelObj) []modelObj { + // Always set, also to nil: what a backend claims about itself on + // /v1/models is never passed through, the configuration decides. + for i := range ms { + ms[i].Capabilities = b.Capabilities + } + return ms } // validateModels filters out entries with an unusable "id". diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 14c8787..a2f9635 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -721,7 +721,7 @@ func asrBackends(t *testing.T) (cfg *config.Config, got map[string]string) { "/v1/audio/transcriptions": func(w http.ResponseWriter, r *http.Request) { b, _ := io.ReadAll(r.Body) got[name] = string(b) - jsonOK(`{"text":"` + name + `"}`)(w, r) + jsonOK(`{"text":"`+name+`"}`)(w, r) }, "/v1/audio/translations": jsonOK(`{"text":"` + name + `"}`), }) @@ -821,3 +821,93 @@ func TestRouting_TranslationsStayOnWhisper(t *testing.T) { t.Errorf("got %s, want whisper", w.Body.String()) } } + +// ── Capabilities ────────────────────────────────────────────────────────────── + +func capabilityConfig(whisperURL, embedURL string) *config.Config { + return buildConfig( + config.Backend{Name: "embedding", BaseURL: embedURL, Enabled: true, Timeout: 5 * time.Second, Models: []string{"BAAI/bge-m3"}, + Capabilities: &config.Capabilities{Tasks: []string{"embed"}}}, + config.Backend{Name: "reranker"}, + config.Backend{Name: "whisper", BaseURL: whisperURL, Enabled: true, Timeout: 5 * time.Second, + Capabilities: &config.Capabilities{Tasks: []string{"transcribe"}, Audio: &config.AudioCapabilities{ + Prompt: true, Hotwords: true, WordTimes: true, MaxSeconds: 900}}}, + ) +} + +func TestModels_CarryConfiguredCapabilities(t *testing.T) { + whisper := mockBackend(t, map[string]http.HandlerFunc{ + // The backend claims capabilities of its own: they must not pass through. + "/v1/models": jsonOK(`{"object":"list","data":[{"id":"whisper-large-v3","object":"model","created":1,"owned_by":"x","capabilities":{"tasks":["lie"]}}]}`), + }) + embed := mockBackend(t, map[string]http.HandlerFunc{"/v1/models": func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(500) }}) + router := buildRouter(t, capabilityConfig(whisper.URL, embed.URL)) + + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + var resp struct { + Data []struct { + ID string `json:"id"` + Capabilities *struct { + Tasks []string `json:"tasks"` + Audio *struct { + Prompt bool `json:"prompt"` + Hotwords bool `json:"hotwords"` + WordTimes bool `json:"word_times"` + MaxSeconds int `json:"max_seconds"` + } `json:"audio"` + } `json:"capabilities"` + } `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil || len(resp.Data) != 2 { + t.Fatalf("%v\n%s", err, w.Body.String()) + } + byID := map[string]int{} + for i, m := range resp.Data { + byID[m.ID] = i + } + wm := resp.Data[byID["whisper-large-v3"]] + if wm.Capabilities == nil || !reflect.DeepEqual(wm.Capabilities.Tasks, []string{"transcribe"}) || wm.Capabilities.Audio == nil || + !wm.Capabilities.Audio.Prompt || !wm.Capabilities.Audio.Hotwords || wm.Capabilities.Audio.MaxSeconds != 900 { + t.Fatalf("whisper: %+v", wm.Capabilities) + } + // The embedding backend failed upstream: static fallback still carries them. + em := resp.Data[byID["BAAI/bge-m3"]] + if em.Capabilities == nil || !reflect.DeepEqual(em.Capabilities.Tasks, []string{"embed"}) || em.Capabilities.Audio != nil { + t.Fatalf("embedding: %+v", em.Capabilities) + } +} + +func TestModels_NoCapabilitiesWhenNotConfigured(t *testing.T) { + whisper := mockBackend(t, map[string]http.HandlerFunc{"/v1/models": jsonOK(modelsJSON("whisper-large-v3"))}) + cfg := buildConfig(config.Backend{}, config.Backend{}, config.Backend{Name: "whisper", BaseURL: whisper.URL, Enabled: true, Timeout: 5 * time.Second}) + w := httptest.NewRecorder() + buildRouter(t, cfg).ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + if strings.Contains(w.Body.String(), "capabilities") { + t.Fatalf("field present without configuration: %s", w.Body.String()) + } +} + +func TestModelByID(t *testing.T) { + whisper := mockBackend(t, map[string]http.HandlerFunc{"/v1/models": jsonOK(modelsJSON("whisper-large-v3"))}) + embed := mockBackend(t, map[string]http.HandlerFunc{"/v1/models": jsonOK(modelsJSON("BAAI/bge-m3"))}) + router := buildRouter(t, capabilityConfig(whisper.URL, embed.URL)) + + for _, id := range []string{"whisper-large-v3", "BAAI/bge-m3"} { // ids may contain a slash + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/models/"+id, nil)) + var m struct { + ID string `json:"id"` + Object string `json:"object"` + Capabilities json.RawMessage `json:"capabilities"` + } + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &m) != nil || m.ID != id || m.Object != "model" || len(m.Capabilities) == 0 { + t.Fatalf("%s: %d %s", id, w.Code, w.Body.String()) + } + } + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/v1/models/nope", nil)) + if w.Code != http.StatusNotFound || !strings.Contains(w.Body.String(), "model_not_found") { + t.Fatalf("unknown id: %d %s", w.Code, w.Body.String()) + } +}