Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ This file provides context for AI coding agents working in this repository.
| Whisper ASR | faster-whisper (Python) | large-v3-turbo | `/v1/audio/transcriptions`, `/v1/audio/translations` | `WHISPER_ENABLED`, `WHISPER_URL` |
| Parakeet ASR | onnx-asr + ONNX Runtime (Python) | parakeet-tdt-0.6b-v3 (int8) | `/v1/audio/transcriptions` when `model` names it | `PARAKEET_ENABLED`, `PARAKEET_URL` |

`/v1/models` returns a merged list from all enabled backends (with static fallback).
`/v1/models` (and `/v1/models/{id}`) returns a merged list from all enabled backends (with static fallback); each entry carries the configured `capabilities` of its backend (`internal/config/capabilities.go`, override with `<NAME>_CAPABILITIES`).

---

Expand Down
24 changes: 23 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ batch tokens) is in [docs/tuning.md](docs/tuning.md); all chart options are comm
| `POST /v1/rerank`, `POST /rerank` | reranker | `/v1/rerank` is rewritten to TEI's `/rerank` |
| `POST /v1/audio/transcriptions` | whisper, parakeet | multipart; `json`, `text`, `verbose_json`, `srt`, `vtt`. Goes to parakeet only when it is enabled and `model` is `parakeet-tdt-0.6b-v3` |
| `POST /v1/audio/translations` | whisper | exposed, but `large-v3-turbo` is not trained for translation |
| `GET /v1/models` | all enabled | merged list incl. `limits` of embedding and reranker |
| `GET /v1/models`, `GET /v1/models/{id}` | all enabled | merged list incl. `limits` of embedding and reranker and the [`capabilities`](#model-capabilities) of every model |
| `GET /healthz`, `GET /readyz` | router | per-backend status, always HTTP 200 (`ok` or `degraded`) |
| `GET :9090/metrics` | router | Prometheus |

Expand All @@ -126,6 +126,28 @@ curl http://localhost:8080/v1/audio/transcriptions -F model=whisper-large-v3-tur
A disabled backend answers `503 backend_disabled`; an unreachable one `502 upstream_error`; one at its
concurrency bound `503 backend_overloaded` with `Retry-After`.

## Model capabilities

Every entry of `GET /v1/models` (and `GET /v1/models/{id}`) carries an additive `capabilities` object so that a client can adapt to the model behind the router instead of guessing from its name. The values come from the router's configuration, never from what a backend claims about itself. Defaults match the bundled servers; replace them per backend with `<NAME>_CAPABILITIES` (JSON, `NAME` one of `EMBEDDING`, `RERANKER`, `WHISPER`, `PARAKEET`) or `router.capabilities.<backend>` in the chart. A malformed value stops the router at start.

```json
{
"id": "parakeet-tdt-0.6b-v3",
"object": "model",
"capabilities": {
"tasks": ["transcribe"],
"audio": {
"prompt": false, "hotwords": false,
"word_times": true, "segment_times": true, "confidence": true,
"auto_language": true, "force_language": false,
"max_seconds": 120, "max_bytes": 26214400
}
}
}
```

`audio.prompt` and `audio.hotwords` tell a client whether a `prompt` or `hotwords` form field has any effect (a backend may accept and ignore it), `max_seconds` and `max_bytes` are per request (0 or absent: not stated), `languages` lists ISO 639-1 codes when the model has a fixed set. Unknown fields must be ignored by clients.

## Components and images

| Component | Image(s) in `ghcr.io/layer87-labs/` | Purpose |
Expand Down
6 changes: 6 additions & 0 deletions deploy/helm/templates/router/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ spec:
- name: PARAKEET_URL
value: "http://{{ include "inference-stack.fullname" . }}-parakeet:{{ .Values.parakeet.service.port }}"
{{- end }}
{{- range $name, $json := .Values.router.capabilities }}
{{- if $json }}
- name: {{ upper $name }}_CAPABILITIES
value: {{ $json | trim | quote }}
{{- end }}
{{- end }}
{{- with .Values.router.extraEnv }}
{{- toYaml . | nindent 8 }}
{{- end }}
Expand Down
7 changes: 7 additions & 0 deletions deploy/helm/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@ router:

healthCheckInterval: "15s"

# Capabilities published on /v1/models and /v1/models/{id} per backend. The
# router has defaults that match the bundled servers; set a backend's key
# (embedding, reranker, whisper, parakeet) to a JSON document to replace
# them, for example when you changed a server limit:
# whisper: '{"tasks":["transcribe"],"audio":{"prompt":true,"hotwords":true,"word_times":true,"segment_times":true,"max_bytes":52428800}}'
capabilities: {}

# Extra environment variables for the router, e.g. to tune per-backend
# timeouts (WHISPER_TIMEOUT, ROUTER_WRITE_TIMEOUT, ...). List of EnvVar.
extraEnv: []
Expand Down
81 changes: 81 additions & 0 deletions internal/config/capabilities.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package config

import (
"encoding/json"
"fmt"
"os"
)

// Capabilities describe what the model(s) behind a backend can do. They are
// published as the "capabilities" field of /v1/models entries, so that a
// client can adapt (send a prompt or not, ask for word timestamps, cut audio
// to the longest accepted length) instead of guessing from the model name.
// The field is additive: clients that do not know it ignore it.
type Capabilities struct {
Tasks []string `json:"tasks,omitempty"` // "embed", "rerank", "transcribe"
Languages []string `json:"languages,omitempty"` // ISO 639-1; empty = not stated
Audio *AudioCapabilities `json:"audio,omitempty"`
}

// AudioCapabilities are the abilities of a transcription backend.
type AudioCapabilities struct {
Prompt bool `json:"prompt"` // accepts a context prompt (`prompt` form field)
PromptMax int `json:"prompt_max,omitempty"` // characters; 0 = not stated
Hotwords bool `json:"hotwords"` // accepts a `hotwords` form field
HotwordsMax int `json:"hotwords_max,omitempty"` // characters; 0 = not stated
WordTimes bool `json:"word_times"` // word timestamps in verbose_json
SegmentTimes bool `json:"segment_times"` // segment timestamps in verbose_json
Confidence bool `json:"confidence"` // per-segment or per-word confidence
AutoLanguage bool `json:"auto_language"` // detects the language itself
ForceLang bool `json:"force_language"` // honours the `language` field
Diarization bool `json:"diarization"` // separates speakers
Streaming bool `json:"streaming"` // takes audio while it is spoken
MaxSeconds int `json:"max_seconds,omitempty"` // longest audio per request; 0 = not stated
MaxBytes int `json:"max_bytes,omitempty"` // largest upload per request; 0 = not stated
Formats []string `json:"formats,omitempty"` // accepted containers; empty = not stated
}

func (c *Capabilities) validate() error {
if a := c.Audio; a != nil && (a.PromptMax < 0 || a.HotwordsMax < 0 || a.MaxSeconds < 0 || a.MaxBytes < 0) {
return fmt.Errorf("negative limit")
}
return nil
}

// capabilitiesFromEnv returns the JSON in the environment variable key,
// or def when it is unset. A malformed value is an error: a typo must not
// silently publish the defaults.
func capabilitiesFromEnv(key string, def *Capabilities) (*Capabilities, error) {
v := os.Getenv(key)
if v == "" {
return def, nil
}
var c Capabilities
if err := json.Unmarshal([]byte(v), &c); err != nil {
return nil, fmt.Errorf("%s: invalid capabilities JSON: %w", key, err)
}
if err := c.validate(); err != nil {
return nil, fmt.Errorf("%s: %w", key, err)
}
return &c, nil
}

// Defaults for the bundled backends, matching what the servers in deploy/ do.
func embeddingCapabilities() *Capabilities { return &Capabilities{Tasks: []string{"embed"}} }
func rerankerCapabilities() *Capabilities { return &Capabilities{Tasks: []string{"rerank"}} }

func whisperCapabilities() *Capabilities {
return &Capabilities{Tasks: []string{"transcribe"}, Audio: &AudioCapabilities{
Prompt: true, PromptMax: 2000, Hotwords: true, HotwordsMax: 2000,
WordTimes: true, SegmentTimes: true, Confidence: true, AutoLanguage: true, ForceLang: true,
MaxBytes: 100 << 20,
}}
}

func parakeetCapabilities() *Capabilities {
return &Capabilities{Tasks: []string{"transcribe"}, Audio: &AudioCapabilities{
// The prompt field is accepted and ignored: the model has no prompt input.
WordTimes: true, SegmentTimes: true, Confidence: true, AutoLanguage: true,
MaxSeconds: 120, MaxBytes: 25 << 20,
}}
}
22 changes: 22 additions & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ type Backend struct {
// under load. 0 or negative disables the limit (unbounded passthrough,
// the previous behavior).
MaxConcurrent int

// Capabilities are published on /v1/models for the models of this
// backend (nil: not published). Defaults for the bundled backends can be
// replaced with <NAME>_CAPABILITIES (JSON).
Capabilities *Capabilities
}

// Config holds all runtime configuration for the inference router.
Expand Down Expand Up @@ -113,6 +118,23 @@ func Load() (*Config, error) {
},
}

for _, c := range []struct {
b *Backend
key string
def *Capabilities
}{
{&cfg.Embedding, "EMBEDDING_CAPABILITIES", embeddingCapabilities()},
{&cfg.Reranker, "RERANKER_CAPABILITIES", rerankerCapabilities()},
{&cfg.Whisper, "WHISPER_CAPABILITIES", whisperCapabilities()},
{&cfg.Parakeet, "PARAKEET_CAPABILITIES", parakeetCapabilities()},
} {
caps, err := capabilitiesFromEnv(c.key, c.def)
if err != nil {
return nil, err
}
c.b.Capabilities = caps
}

if err := cfg.validate(); err != nil {
return nil, err
}
Expand Down
57 changes: 57 additions & 0 deletions internal/config/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package config

import (
"reflect"
"strings"
"testing"
)

func setEnabled(t *testing.T) {
t.Helper()
t.Setenv("WHISPER_ENABLED", "true")
t.Setenv("WHISPER_URL", "http://whisper:8000")
t.Setenv("PARAKEET_ENABLED", "true")
t.Setenv("PARAKEET_URL", "http://parakeet:8000")
}

func TestLoad_CapabilityDefaults(t *testing.T) {
setEnabled(t)
cfg, err := Load()
if err != nil {
t.Fatal(err)
}
w, p := cfg.Whisper.Capabilities.Audio, cfg.Parakeet.Capabilities.Audio
if !w.Prompt || !w.Hotwords || !w.WordTimes || !w.ForceLang || w.MaxBytes != 100<<20 {
t.Errorf("whisper: %+v", w)
}
// The second backend differs: no prompt, no hotwords, a short audio limit.
if p.Prompt || p.Hotwords || !p.WordTimes || p.ForceLang || p.MaxSeconds != 120 || p.MaxBytes != 25<<20 {
t.Errorf("parakeet: %+v", p)
}
if !reflect.DeepEqual(cfg.Embedding.Capabilities.Tasks, []string{"embed"}) || !reflect.DeepEqual(cfg.Reranker.Capabilities.Tasks, []string{"rerank"}) {
t.Errorf("tasks: %+v %+v", cfg.Embedding.Capabilities, cfg.Reranker.Capabilities)
}
}

func TestLoad_CapabilityOverride(t *testing.T) {
setEnabled(t)
t.Setenv("WHISPER_CAPABILITIES", `{"tasks":["transcribe"],"languages":["de","en"],"audio":{"prompt":false,"hotwords":true,"max_seconds":300}}`)
cfg, err := Load()
if err != nil {
t.Fatal(err)
}
c := cfg.Whisper.Capabilities
if !reflect.DeepEqual(c.Languages, []string{"de", "en"}) || c.Audio.Prompt || !c.Audio.Hotwords || c.Audio.MaxSeconds != 300 {
t.Errorf("%+v %+v", c, c.Audio)
}
}

func TestLoad_BadCapabilitiesFail(t *testing.T) {
setEnabled(t)
for name, v := range map[string]string{"not json": `{"audio":`, "negative": `{"audio":{"max_bytes":-1}}`} {
t.Setenv("PARAKEET_CAPABILITIES", v)
if _, err := Load(); err == nil || !strings.Contains(err.Error(), "PARAKEET_CAPABILITIES") {
t.Errorf("%s: got %v", name, err)
}
}
}
52 changes: 45 additions & 7 deletions internal/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,9 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
case path == "/v1/models" || path == "/v1/models/":
r.handleModels(w, req)

case strings.HasPrefix(path, "/v1/models/") && req.Method == http.MethodGet:
r.handleModel(w, strings.TrimPrefix(path, "/v1/models/"))

case strings.HasPrefix(path, "/v1/embeddings") || path == "/embed" || path == "/embed_sparse":
if strings.HasPrefix(path, "/v1/embeddings") {
r.stripAdvertisedModel(req, r.cfg.Embedding)
Expand Down Expand Up @@ -239,6 +242,9 @@ type modelObj struct {
Created int64 `json:"created"`
OwnedBy string `json:"owned_by"`
Limits *modelLimits `json:"limits,omitempty"`
// Capabilities say what the model can do (prompt, word timestamps, longest
// audio, ...), from the backend's configuration. Additive field.
Capabilities *config.Capabilities `json:"capabilities,omitempty"`
}

// modelLimits are the request-size limits of the running backend, read from
Expand Down Expand Up @@ -392,7 +398,31 @@ type modelsResp struct {
}

// handleModels returns a merged model list from all enabled backends.
func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(modelsResp{Object: "list", Data: r.collectModels()})
}

// handleModel serves GET /v1/models/{id}: one entry of the merged list, in the
// OpenAI shape, or a 404 error object.
func (r *Router) handleModel(w http.ResponseWriter, id string) {
for _, m := range r.collectModels() {
if m.ID == id {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(m)
return
}
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusNotFound)
_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{
"message": "model not found", "type": "invalid_request_error", "code": "model_not_found",
}})
}

// collectModels merges the model lists of all enabled backends and attaches
// each backend's limits and capabilities.
func (r *Router) collectModels() []modelObj {
var models []modelObj
client := &http.Client{Timeout: 5 * time.Second}

Expand All @@ -401,20 +431,20 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
if err != nil {
r.log.Warn("failed to fetch models from backend",
zap.String("backend", b.Name), zap.Error(err))
models = append(models, r.staticModels(b)...)
models = append(models, r.withCapabilities(b, r.staticModels(b))...)
continue
}

body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<16))
_ = resp.Body.Close()
if err != nil || resp.StatusCode != http.StatusOK {
models = append(models, r.staticModels(b)...)
models = append(models, r.withCapabilities(b, r.staticModels(b))...)
continue
}

var upstreamResp modelsResp
if err := json.Unmarshal(body, &upstreamResp); err != nil {
models = append(models, r.staticModels(b)...)
models = append(models, r.withCapabilities(b, r.staticModels(b))...)
continue
}

Expand All @@ -427,11 +457,19 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
valid[i].Limits = limits
}
}
models = append(models, valid...)
models = append(models, r.withCapabilities(b, valid)...)
}
return models
}

w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(modelsResp{Object: "list", Data: models})
// withCapabilities sets the backend's configured capabilities on its models.
func (r *Router) withCapabilities(b config.Backend, ms []modelObj) []modelObj {
// Always set, also to nil: what a backend claims about itself on
// /v1/models is never passed through, the configuration decides.
for i := range ms {
ms[i].Capabilities = b.Capabilities
}
return ms
}

// validateModels filters out entries with an unusable "id".
Expand Down
Loading
Loading