Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,32 @@ jobs:

- name: Test
run: pytest -q deploy/whisper

# ── Python: parakeet server tests (fake model, real audio decoding) ───────
parakeet:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7

- uses: actions/setup-python@v6
with:
python-version: "3.13"

- name: Install test dependencies
run: |
grep -E '^(numpy|av|fastapi|python-multipart)==' deploy/parakeet/requirements.in > /tmp/test-req.txt
pip install -r /tmp/test-req.txt httpx pytest

- name: Test
run: pytest -q deploy/parakeet

# ── Helm: render the chart with optional backends off and on ──────────────
chart:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7

- uses: azure/setup-helm@v4

- name: Test
run: scripts/test-chart.sh
56 changes: 55 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ on:
workflow_dispatch:
inputs:
components:
description: "Components to release even if nothing changed (comma-separated: router,tei,embedding-model,reranker-model,whisper,chart or 'all')"
description: "Components to release even if nothing changed (comma-separated: router,tei,embedding-model,reranker-model,whisper,parakeet,chart or 'all')"
required: false
default: ""
type: string
Expand Down Expand Up @@ -43,6 +43,7 @@ jobs:
embedding-model: ${{ steps.decide.outputs.embedding-model }}
reranker-model: ${{ steps.decide.outputs.reranker-model }}
whisper: ${{ steps.decide.outputs.whisper }}
parakeet: ${{ steps.decide.outputs.parakeet }}
chart: ${{ steps.decide.outputs.chart }}
steps:
- uses: actions/checkout@v7
Expand Down Expand Up @@ -70,6 +71,9 @@ jobs:
- name: Changed (whisper)
id: whisper
run: relctl release changed --component whisper
- name: Changed (parakeet)
id: parakeet
run: relctl release changed --component parakeet
- name: Changed (chart)
id: chart
run: relctl release changed --component chart
Expand All @@ -83,6 +87,7 @@ jobs:
CHANGED_EMBEDDING_MODEL: ${{ steps.embedding-model.outputs.RELCTL_CHANGED }}
CHANGED_RERANKER_MODEL: ${{ steps.reranker-model.outputs.RELCTL_CHANGED }}
CHANGED_WHISPER: ${{ steps.whisper.outputs.RELCTL_CHANGED }}
CHANGED_PARAKEET: ${{ steps.parakeet.outputs.RELCTL_CHANGED }}
CHANGED_CHART: ${{ steps.chart.outputs.RELCTL_CHANGED }}
run: |
decide() { # name changed
Expand All @@ -99,6 +104,7 @@ jobs:
decide embedding-model "${CHANGED_EMBEDDING_MODEL}"
decide reranker-model "${CHANGED_RERANKER_MODEL}"
decide whisper "${CHANGED_WHISPER}"
decide parakeet "${CHANGED_PARAKEET}"
decide chart "${CHANGED_CHART}"

# ── router ────────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -382,6 +388,54 @@ jobs:
relctl-version: ${{ env.RELCTL_TOOL_VERSION }}
run: relctl release publish --component whisper --release-id ${{ needs.whisper-release.outputs.release-id }}

# ── parakeet (image tag: tdt-0.6b-v3-<version> and latest) ──────────────
parakeet-release:
needs: changes
if: needs.changes.outputs.parakeet == 'true'
uses: ./.github/workflows/component-release.yml
with:
component: parakeet

parakeet-build:
needs: parakeet-release
if: needs.parakeet-release.outputs.release-id != ''
runs-on: ubuntu-latest
env:
VERSION: ${{ needs.parakeet-release.outputs.version }}
MODEL_TAG: tdt-0.6b-v3-${{ needs.parakeet-release.outputs.version }}
steps:
- uses: actions/checkout@v7

- name: Log in to GHCR
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build and push parakeet
run: |
docker build \
-f deploy/Containerfile.parakeet \
-t ${{ env.REGISTRY }}/parakeet:${MODEL_TAG} \
-t ${{ env.REGISTRY }}/parakeet:latest \
.
docker push ${{ env.REGISTRY }}/parakeet:${MODEL_TAG}
docker push ${{ env.REGISTRY }}/parakeet:latest

- name: Install cosign
uses: sigstore/cosign-installer@v3

- name: Sign image (keyless via OIDC, by digest)
run: scripts/sign-image.sh "${{ env.REGISTRY }}/parakeet:${MODEL_TAG}"

- name: Publish release
uses: layer87-labs/relctl-action@v1
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
relctl-version: ${{ env.RELCTL_TOOL_VERSION }}
run: relctl release publish --component parakeet --release-id ${{ needs.parakeet-release.outputs.release-id }}

# ── chart (OCI → ghcr.io) ─────────────────────────────────────────────────
# Runs after router and tei so that the chart pins the versions just released.
chart-release:
Expand Down
5 changes: 5 additions & 0 deletions .relctl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ components:
- "deploy/whisper/**"
- "deploy/Containerfile.whisper"
version_scheme: calver
parakeet:
paths:
- "deploy/parakeet/**"
- "deploy/Containerfile.parakeet"
version_scheme: calver
# The chart pins the router and tei image tags when it is packaged, so it is
# also released when one of those components is.
chart:
Expand Down
2 changes: 2 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ This file provides context for AI coding agents working in this repository.
| Embedding | TEI (Rust) | BAAI/bge-m3 | `/v1/embeddings`, `/embed`, `/embed_sparse` | `EMBEDDING_ENABLED`, `EMBEDDING_URL` |
| Reranker | TEI (Rust, ONNX) | cross-encoder/mmarco-mMiniLMv2-L12-H384-v1 (default), BAAI/bge-reranker-v2-m3 | `/v1/rerank`, `/rerank` | `RERANKER_ENABLED`, `RERANKER_URL` |
| Whisper ASR | faster-whisper (Python) | large-v3-turbo | `/v1/audio/transcriptions`, `/v1/audio/translations` | `WHISPER_ENABLED`, `WHISPER_URL` |
| Parakeet ASR | onnx-asr + ONNX Runtime (Python) | parakeet-tdt-0.6b-v3 (int8) | `/v1/audio/transcriptions` when `model` names it | `PARAKEET_ENABLED`, `PARAKEET_URL` |

`/v1/models` returns a merged list from all enabled backends (with static fallback).

Expand Down Expand Up @@ -41,6 +42,7 @@ deploy/
Containerfile.reranker-model-init Reranker model init container
Containerfile.reranker-server FlagEmbedding FastAPI server (/v1/rerank)
Containerfile.whisper Whisper ASR server (deploy/whisper/server.py) with model baked in
Containerfile.parakeet Parakeet ASR server (deploy/parakeet/server.py), Chainguard Python base, model baked in
whisper/
server.py faster-whisper FastAPI server (/v1/audio/transcriptions)
reranker/
Expand Down
20 changes: 17 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ WHISPER_MODEL ?= $(shell grep '^ARG WHISPER_MODEL=' deploy/Containerfile.whisp
docker/tei-base docker/tei-runtime \
docker/tei-model-init-embedding \
docker/tei-reranker-model-init \
docker/whisper \
docker/whisper docker/parakeet push/parakeet \
push push/router push/tei push/reranker push/whisper \
versions

Expand Down Expand Up @@ -127,12 +127,26 @@ docker/whisper:
-t $(REGISTRY)/whisper:$(WHISPER_MODEL)-$(WHISPER_IMAGE_VERSION) \
.

## ── Docker: Parakeet ────────────────────────────────────────────────────────

PARAKEET_IMAGE_VERSION ?= $(VERSION)

# parakeet — tag = tdt-0.6b-v3-<parakeet version>
docker/parakeet:
docker build \
-f deploy/Containerfile.parakeet \
-t $(REGISTRY)/parakeet:tdt-0.6b-v3-$(PARAKEET_IMAGE_VERSION) \
.

push/parakeet:
docker push $(REGISTRY)/parakeet:tdt-0.6b-v3-$(PARAKEET_IMAGE_VERSION)

## ── Build all ─────────────────────────────────────────────────────────────────

docker: docker/router docker/tei-base docker/tei-runtime \
docker/tei-model-init-embedding \
docker/tei-reranker-model-init \
docker/whisper
docker/whisper docker/parakeet

## ── Push ─────────────────────────────────────────────────────────────────────

Expand All @@ -150,7 +164,7 @@ push/reranker:
push/whisper:
docker push $(REGISTRY)/whisper:$(WHISPER_MODEL)-$(WHISPER_IMAGE_VERSION)

push: push/router push/tei push/reranker push/whisper
push: push/router push/tei push/reranker push/whisper push/parakeet

## ── Info ──────────────────────────────────────────────────────────────────────

Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ batch tokens) is in [docs/tuning.md](docs/tuning.md); all chart options are comm
| `POST /v1/embeddings` | embedding | OpenAI-style. The advertised model id `BAAI/bge-m3` may be sent as `model`. |
| `POST /embed`, `POST /embed_sparse` | embedding | TEI native (dense and sparse) |
| `POST /v1/rerank`, `POST /rerank` | reranker | `/v1/rerank` is rewritten to TEI's `/rerank` |
| `POST /v1/audio/transcriptions` | whisper | multipart; `json`, `text`, `verbose_json`, `srt`, `vtt` |
| `POST /v1/audio/transcriptions` | whisper, parakeet | multipart; `json`, `text`, `verbose_json`, `srt`, `vtt`. Goes to parakeet only when it is enabled and `model` is `parakeet-tdt-0.6b-v3` |
| `POST /v1/audio/translations` | whisper | exposed, but `large-v3-turbo` is not trained for translation |
| `GET /v1/models` | all enabled | merged list incl. `limits` of embedding and reranker |
| `GET /healthz`, `GET /readyz` | router | per-backend status, always HTTP 200 (`ok` or `degraded`) |
Expand All @@ -135,6 +135,7 @@ concurrency bound `503 backend_overloaded` with `Retry-After`.
| embedding-model | `tei-model-init` | `BAAI/bge-m3` baked in, copies to the volume |
| reranker-model | `tei-reranker-model-init` | Reranker as ONNX: `cross-encoder/mmarco-mMiniLMv2-L12-H384-v1` (default) or `BAAI/bge-reranker-v2-m3` |
| whisper | `whisper` | faster-whisper server, `large-v3-turbo` baked in |
| parakeet | `parakeet` | Fast transcription of short audio: NVIDIA Parakeet-TDT-0.6B-v3 (int8 ONNX, CC-BY-4.0) on ONNX Runtime, baked in |
| chart | `charts/inference-stack` (OCI) | Helm chart |

Why the default reranker is the small model: on 4 CPUs it needs 0.76 to 0.85 s for 20 documents of 800
Expand Down
1 change: 1 addition & 0 deletions cmd/router/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func main() {
zap.Bool("embedding_enabled", cfg.Embedding.Enabled),
zap.Bool("reranker_enabled", cfg.Reranker.Enabled),
zap.Bool("whisper_enabled", cfg.Whisper.Enabled),
zap.Bool("parakeet_enabled", cfg.Parakeet.Enabled),
)

ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
Expand Down
88 changes: 88 additions & 0 deletions deploy/Containerfile.parakeet
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
# syntax=docker/dockerfile:1
#
# Containerfile.parakeet — Parakeet ASR HTTP server (CPU), model baked in
#
# OpenAI-compatible POST /v1/audio/transcriptions served by
# deploy/parakeet/server.py on top of onnx-asr and ONNX Runtime (CPU). A fast
# path for short recordings; Whisper stays the long-form backend.
#
# Model: NVIDIA Parakeet-TDT-0.6B-v3, int8 ONNX export by istupakov
# (https://huggingface.co/istupakov/parakeet-tdt-0.6b-v3-onnx), converted from
# https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3.
# License of the model: CC-BY-4.0 (attribution in /opt/model/NOTICE and in
# the image description). The files are downloaded at a fixed commit and every
# file is checked against its SHA-256; a moved or changed source fails the
# build. Only ONNX and text files, no pickle. No download at runtime.
# To update: pick a new commit, set PARAKEET_MODEL_REVISION and the SHA-256
# values below (Hugging Face file page, "Git LFS Details", or sha256sum).
#
# Base: Chainguard wolfi-base (glibc) with Python 3.13 from the Wolfi
# repository. The public cgr.dev/chainguard/python image only ships the
# newest Python (3.15 at the time of writing), for which ONNX Runtime has no
# wheels yet; wolfi-base lets the build pick the Python minor version.
# BASE_IMAGE can point to a mirror of cgr.dev/chainguard.
#
# Hardened: non-root (uid 65532), no compilers or pip in the runtime image,
# no network access needed at runtime.

ARG BASE_IMAGE=cgr.dev/chainguard/wolfi-base:latest@sha256:6b818b54724b30d095379a7471d9a9b6428a51ff52be188676d1d6e5ea67a1e4
ARG PYTHON=3.13

FROM ${BASE_IMAGE} AS build
ARG PYTHON

ARG PARAKEET_MODEL_REPO=istupakov/parakeet-tdt-0.6b-v3-onnx
ARG PARAKEET_MODEL_REVISION=8f23f0c03c8761650bdb5b40aaf3e40d2c15f1ce

RUN apk add --no-cache python-${PYTHON} py${PYTHON}-pip
COPY deploy/parakeet/requirements.txt /tmp/requirements.txt
RUN python${PYTHON} -m venv /venv && \
/venv/bin/pip install --no-cache-dir --require-hashes --only-binary=:all: -r /tmp/requirements.txt && \
rm /tmp/requirements.txt

# Model files and their SHA-256 (int8 encoder and decoder; the NeMo
# preprocessor ships inside onnx-asr).
COPY <<'EOF' /tmp/model.sha256
666903c76b9798caf2c210afd4f6cd60b08a8dbf9800ec8d7a3bc0d2148ac466 config.json
eea7483ee3d1a30375daedc8ed83e3960c91b098812127a0d99d1c8977667a70 decoder_joint-model.int8.onnx
6139d2fa7e1b086097b277c7149725edbab89cc7c7ae64b23c741be4055aff09 encoder-model.int8.onnx
d58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d vocab.txt
EOF

RUN mkdir -p /opt/model && cd /opt/model && \
for f in $(awk '{print $2}' /tmp/model.sha256); do \
/venv/bin/python -c "import sys, urllib.request; urllib.request.urlretrieve(sys.argv[1], sys.argv[2])" \
"https://huggingface.co/${PARAKEET_MODEL_REPO}/resolve/${PARAKEET_MODEL_REVISION}/${f}" "${f}" || exit 1; \
done && \
sha256sum -c /tmp/model.sha256

COPY <<'EOF' /opt/model/NOTICE
Parakeet-TDT-0.6B-v3 by NVIDIA (https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3),
ONNX export by istupakov (https://huggingface.co/istupakov/parakeet-tdt-0.6b-v3-onnx).
Licensed under CC-BY-4.0 (https://creativecommons.org/licenses/by/4.0/).
Changes: none to the weights; int8-quantized ONNX files are redistributed as published.
EOF

FROM ${BASE_IMAGE}
ARG PYTHON

RUN apk add --no-cache python-${PYTHON}

COPY --from=build /venv /venv
COPY --from=build /opt/model /opt/model
COPY deploy/parakeet/server.py /app/server.py

ENV PATH=/venv/bin:$PATH \
MODEL_PATH=/opt/model \
MODEL_NAME=parakeet-tdt-0.6b-v3 \
PORT=9000

WORKDIR /app
USER 65532
EXPOSE 9000

ENTRYPOINT ["/venv/bin/python", "/app/server.py"]

LABEL org.opencontainers.image.source="https://github.com/layer87-labs/inference-stack"
LABEL org.opencontainers.image.description="Parakeet ASR HTTP server (CPU, ONNX Runtime) — OpenAI-compatible /v1/audio/transcriptions. Model: NVIDIA Parakeet-TDT-0.6B-v3 (int8 ONNX export by istupakov), CC-BY-4.0"
LABEL org.opencontainers.image.licenses="Apache-2.0 AND CC-BY-4.0"
Loading
Loading