diff --git a/mkdocs/docs/concepts/services.md b/mkdocs/docs/concepts/services.md index a7d24fe9a..0703ce083 100644 --- a/mkdocs/docs/concepts/services.md +++ b/mkdocs/docs/concepts/services.md @@ -170,7 +170,7 @@ both are set per group. Router sits in front of inference backend workers (SGLang, vLLM, TensorRT-LLM) and decides which worker serves each request based on routing policy. -Below is an example for running `Qwen/Qwen3.8-Flash-Next` on `H200`: +Below are examples for running `Qwen/Qwen3.8-Flash-Next` with SMG and `zai-org/GLM-4.5-Air-FP8` with Dynamo on `H200`: === "SMG" @@ -234,77 +234,64 @@ Below is an example for running `Qwen/Qwen3.8-Flash-Next` on `H200`: ```yaml type: service - name: qwen38-flash-next-dynamo + name: glm-45-air-dynamo env: - HF_TOKEN - - MODEL_ID=Qwen/Qwen3.8-Flash-Next + - MODEL_ID=zai-org/GLM-4.5-Air-FP8 groups: - - replicas: 1 - docker: true + # The frontend and the workers discover each other via etcd + - name: etcd + replicas: 1 commands: - - apt-get update - - apt-get install -y python3-dev python3-venv - - python3 -m venv ~/dyn-venv - - source ~/dyn-venv/bin/activate - - pip install -U pip - - pip install "ai-dynamo[sglang]==1.1.1" - - git clone https://github.com/ai-dynamo/dynamo.git - # Brings up the NATS / etcd compose stack and runs the Dynamo HTTP frontend. - - docker compose -f dynamo/dev/docker-compose.yml up -d - | - python3 -m dynamo.frontend \ - --http-host 0.0.0.0 --http-port 8000 \ - --discovery-backend etcd --router-mode kv \ - --kv-cache-block-size 64 + curl -fsSL https://github.com/etcd-io/etcd/releases/download/v3.5.33/etcd-v3.5.33-linux-amd64.tar.gz \ + | tar -xz -C /usr/local/bin --strip-components=1 etcd-v3.5.33-linux-amd64/etcd + - | + etcd \ + --data-dir /tmp/etcd \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://$DSTACK_MASTER_NODE_IP:2379 resources: - cpu: 4 + cpu: 2 + + - name: frontend + replicas: 1 + # The vLLM image provides vLLM's GLM tool call and reasoning parsers + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.1 + commands: + - export ETCD_ENDPOINTS=http://${{ groups[0].replicas[0].IP_ADDRESS }}:2379 + - | + python3 -m dynamo.frontend \ + --http-port 8000 \ + --router-mode kv \ + --dyn-chat-processor vllm \ + --enable-auto-tool-choice \ + --tool-call-parser glm45 \ + --reasoning-parser glm45 router: type: dynamo + resources: + cpu: 4 - - replicas: 2 - python: "3.12" - nvcc: true + - name: worker + replicas: 2 + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.1 commands: - # dstack injects DSTACK_ROUTER_INTERNAL_IP after the router replica - # is provisioned. Compose the etcd/NATS endpoints from it. - - export ETCD_ENDPOINTS="http://$DSTACK_ROUTER_INTERNAL_IP:2379" - - export NATS_SERVER="nats://$DSTACK_ROUTER_INTERNAL_IP:4222" - # Set to enable /health endpoint required by dstack probes. - - export DYN_SYSTEM_PORT="8000" - # Wait until the router's etcd and NATS ports are actually accepting connections. - - | - until curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:2379/health" \ - && curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:8222/healthz"; do - echo "waiting for etcd/NATS on $DSTACK_ROUTER_INTERNAL_IP..."; sleep 3 - done - - pip install "ai-dynamo[sglang]==1.1.1" + - export ETCD_ENDPOINTS=http://${{ groups[0].replicas[0].IP_ADDRESS }}:2379 + # vLLM binds the KV events endpoint only if it's a wildcard address, otherwise it connects - | - python3 -m dynamo.sglang \ - --model-path $MODEL_ID \ - --served-model-name $MODEL_ID \ - --discovery-backend etcd \ - --host 0.0.0.0 \ - --page-size 64 \ - --tp $DSTACK_GPUS_NUM \ - --ep $DSTACK_GPUS_NUM \ - --mem-fraction-static 0.85 \ - --chunked-prefill-size 8192 \ - --linear-attn-prefill-backend flashinfer \ - --linear-attn-decode-backend flashinfer \ - --mamba-ssm-dtype bfloat16 \ - --reasoning-parser auto + python3 -m dynamo.vllm \ + --model $MODEL_ID \ + --max-model-len 65536 \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' resources: - gpu: H200:4 + gpu: H200 + disk: 200GB.. port: 8000 - model: Qwen/Qwen3.8-Flash-Next - - probes: - - type: http - url: /health - interval: 15s + model: zai-org/GLM-4.5-Air-FP8 ``` @@ -418,84 +405,77 @@ Below is an example for running `zai-org/GLM-4.5-Air-FP8` on `H200`: - MODEL_ID=zai-org/GLM-4.5-Air-FP8 groups: - - replicas: 1 - docker: true + # The frontend and the workers discover each other via etcd + - name: etcd + replicas: 1 commands: - - apt-get update - - apt-get install -y python3-dev python3-venv - - python3 -m venv ~/dyn-venv - - source ~/dyn-venv/bin/activate - - pip install -U pip - - pip install "ai-dynamo[sglang]==1.1.1" - - git clone https://github.com/ai-dynamo/dynamo.git - # Brings up the NATS / etcd compose stack and runs the Dynamo HTTP frontend. - - docker compose -f dynamo/dev/docker-compose.yml up -d - | - python3 -m dynamo.frontend \ - --http-host 0.0.0.0 --http-port 8000 \ - --discovery-backend etcd --router-mode kv \ - --kv-cache-block-size 64 + curl -fsSL https://github.com/etcd-io/etcd/releases/download/v3.5.33/etcd-v3.5.33-linux-amd64.tar.gz \ + | tar -xz -C /usr/local/bin --strip-components=1 etcd-v3.5.33-linux-amd64/etcd + - | + etcd \ + --data-dir /tmp/etcd \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://$DSTACK_MASTER_NODE_IP:2379 resources: - cpu: 4 + cpu: 2 + + - name: frontend + replicas: 1 + # The vLLM image provides vLLM's GLM tool call and reasoning parsers + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.1 + commands: + - export ETCD_ENDPOINTS=http://${{ groups[0].replicas[0].IP_ADDRESS }}:2379 + - | + python3 -m dynamo.frontend \ + --http-port 8000 \ + --router-mode kv \ + --dyn-chat-processor vllm \ + --enable-auto-tool-choice \ + --tool-call-parser glm45 \ + --reasoning-parser glm45 router: type: dynamo + resources: + cpu: 4 - - replicas: 4 - python: "3.12" - nvcc: true + # Only prefill workers publish KV events, as Dynamo's decode routing ignores prefix overlap + - name: prefill-worker + replicas: 4 + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.1 commands: - # dstack injects DSTACK_ROUTER_INTERNAL_IP after the router replica - # is provisioned. Compose the etcd/NATS endpoints from it. - - export ETCD_ENDPOINTS="http://$DSTACK_ROUTER_INTERNAL_IP:2379" - - export NATS_SERVER="nats://$DSTACK_ROUTER_INTERNAL_IP:4222" - # Set to enable /health endpoint required by dstack probes. - - export DYN_SYSTEM_PORT="8000" - # Wait until the router's etcd and NATS are healthy. + - export ETCD_ENDPOINTS=http://${{ groups[0].replicas[0].IP_ADDRESS }}:2379 + - export VLLM_NIXL_SIDE_CHANNEL_HOST=$DSTACK_MASTER_NODE_IP + # vLLM binds the KV events endpoint only if it's a wildcard address, otherwise it connects - | - until curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:2379/health" \ - && curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:8222/healthz"; do - echo "waiting for etcd/NATS on $DSTACK_ROUTER_INTERNAL_IP..."; sleep 3 - done - - pip install "ai-dynamo[sglang]==1.1.1" - - | - python3 -m dynamo.sglang \ - --model-path $MODEL_ID --served-model-name $MODEL_ID \ - --discovery-backend etcd --host 0.0.0.0 \ - --page-size 64 \ - --disaggregation-mode prefill --disaggregation-transfer-backend nixl + python3 -m dynamo.vllm \ + --model $MODEL_ID \ + --max-model-len 65536 \ + --disaggregation-mode prefill \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_producer"}' \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' resources: gpu: H200 + disk: 200GB.. - - replicas: 8 - python: "3.12" - nvcc: true + - name: decode-worker + replicas: 8 + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.1 commands: - - export ETCD_ENDPOINTS="http://$DSTACK_ROUTER_INTERNAL_IP:2379" - - export NATS_SERVER="nats://$DSTACK_ROUTER_INTERNAL_IP:4222" - - export DYN_SYSTEM_PORT="8000" - - | - until curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:2379/health" \ - && curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:8222/healthz"; do - echo "waiting for etcd/NATS on $DSTACK_ROUTER_INTERNAL_IP..."; sleep 3 - done - - pip install "ai-dynamo[sglang]==1.1.1" + - export ETCD_ENDPOINTS=http://${{ groups[0].replicas[0].IP_ADDRESS }}:2379 + - export VLLM_NIXL_SIDE_CHANNEL_HOST=$DSTACK_MASTER_NODE_IP - | - python3 -m dynamo.sglang \ - --model-path $MODEL_ID --served-model-name $MODEL_ID \ - --discovery-backend etcd --host 0.0.0.0 \ - --page-size 64 \ - --disaggregation-mode decode --disaggregation-transfer-backend nixl + python3 -m dynamo.vllm \ + --model $MODEL_ID \ + --max-model-len 65536 \ + --disaggregation-mode decode \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_consumer"}' resources: gpu: H200 + disk: 200GB.. port: 8000 model: zai-org/GLM-4.5-Air-FP8 - - # Custom probe is required for PD disaggregation. - probes: - - type: http - url: /health - interval: 15s ``` diff --git a/mkdocs/docs/examples/inference/dynamo.md b/mkdocs/docs/examples/inference/dynamo.md index 5c9fecc23..998b00b47 100644 --- a/mkdocs/docs/examples/inference/dynamo.md +++ b/mkdocs/docs/examples/inference/dynamo.md @@ -12,7 +12,7 @@ This example shows how to deploy `zai-org/GLM-4.5-Air-FP8` using ## Apply a configuration Here's an example of a service that deploys `zai-org/GLM-4.5-Air-FP8` using -Dynamo with PD disaggregation. +Dynamo with PD disaggregation and vLLM workers.
@@ -25,95 +25,96 @@ env: - MODEL_ID=zai-org/GLM-4.5-Air-FP8 groups: - - replicas: 1 - docker: true + # The frontend and the workers discover each other via etcd + - name: etcd + replicas: 1 commands: - - apt-get update - - apt-get install -y python3-dev python3-venv - - python3 -m venv ~/dyn-venv - - source ~/dyn-venv/bin/activate - - pip install -U pip - - pip install "ai-dynamo[sglang]==1.1.1" - - git clone https://github.com/ai-dynamo/dynamo.git - # Brings up the NATS / etcd compose stack and runs the Dynamo HTTP frontend. - - docker compose -f dynamo/dev/docker-compose.yml up -d - | - python3 -m dynamo.frontend \ - --http-host 0.0.0.0 --http-port 8000 \ - --discovery-backend etcd --router-mode kv \ - --kv-cache-block-size 64 + curl -fsSL https://github.com/etcd-io/etcd/releases/download/v3.5.33/etcd-v3.5.33-linux-amd64.tar.gz \ + | tar -xz -C /usr/local/bin --strip-components=1 etcd-v3.5.33-linux-amd64/etcd + - | + etcd \ + --data-dir /tmp/etcd \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://$DSTACK_MASTER_NODE_IP:2379 resources: - cpu: 4 + cpu: 2 + + - name: frontend + replicas: 1 + # The vLLM image provides vLLM's GLM tool call and reasoning parsers + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.1 + commands: + - export ETCD_ENDPOINTS=http://${{ groups[0].replicas[0].IP_ADDRESS }}:2379 + - | + python3 -m dynamo.frontend \ + --http-port 8000 \ + --router-mode kv \ + --dyn-chat-processor vllm \ + --enable-auto-tool-choice \ + --tool-call-parser glm45 \ + --reasoning-parser glm45 router: type: dynamo + resources: + cpu: 4 - - replicas: 1..4 + # Only prefill workers publish KV events, as Dynamo's decode routing ignores prefix overlap + - name: prefill-worker + replicas: 1..4 scaling: metric: rps target: 3 - python: "3.12" - nvcc: true + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.1 commands: - # dstack injects DSTACK_ROUTER_INTERNAL_IP after the router replica - # is provisioned. Compose the etcd/NATS endpoints from it. - - export ETCD_ENDPOINTS="http://$DSTACK_ROUTER_INTERNAL_IP:2379" - - export NATS_SERVER="nats://$DSTACK_ROUTER_INTERNAL_IP:4222" - # Set to enable /health endpoint required by dstack probes. - - export DYN_SYSTEM_PORT="8000" - # Wait until the router's etcd and NATS are healthy. - - | - until curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:2379/health" \ - && curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:8222/healthz"; do - echo "waiting for etcd/NATS on $DSTACK_ROUTER_INTERNAL_IP..."; sleep 3 - done - - pip install "ai-dynamo[sglang]==1.1.1" + - export ETCD_ENDPOINTS=http://${{ groups[0].replicas[0].IP_ADDRESS }}:2379 + - export VLLM_NIXL_SIDE_CHANNEL_HOST=$DSTACK_MASTER_NODE_IP + # vLLM binds the KV events endpoint only if it's a wildcard address, otherwise it connects - | - python3 -m dynamo.sglang \ - --model-path $MODEL_ID --served-model-name $MODEL_ID \ - --discovery-backend etcd --host 0.0.0.0 \ - --page-size 64 \ - --disaggregation-mode prefill --disaggregation-transfer-backend nixl + python3 -m dynamo.vllm \ + --model $MODEL_ID \ + --max-model-len 65536 \ + --disaggregation-mode prefill \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_producer"}' \ + --kv-events-config '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' resources: gpu: H200 + disk: 200GB.. - - replicas: 1..8 + - name: decode-worker + replicas: 1..8 scaling: metric: rps target: 2 - python: "3.12" - nvcc: true + image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.5.1 commands: - - export ETCD_ENDPOINTS="http://$DSTACK_ROUTER_INTERNAL_IP:2379" - - export NATS_SERVER="nats://$DSTACK_ROUTER_INTERNAL_IP:4222" - - export DYN_SYSTEM_PORT="8000" + - export ETCD_ENDPOINTS=http://${{ groups[0].replicas[0].IP_ADDRESS }}:2379 + - export VLLM_NIXL_SIDE_CHANNEL_HOST=$DSTACK_MASTER_NODE_IP - | - until curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:2379/health" \ - && curl -fsS "http://$DSTACK_ROUTER_INTERNAL_IP:8222/healthz"; do - echo "waiting for etcd/NATS on $DSTACK_ROUTER_INTERNAL_IP..."; sleep 3 - done - - pip install "ai-dynamo[sglang]==1.1.1" - - | - python3 -m dynamo.sglang \ - --model-path $MODEL_ID --served-model-name $MODEL_ID \ - --discovery-backend etcd --host 0.0.0.0 \ - --page-size 64 \ - --disaggregation-mode decode --disaggregation-transfer-backend nixl + python3 -m dynamo.vllm \ + --model $MODEL_ID \ + --max-model-len 65536 \ + --disaggregation-mode decode \ + --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_consumer"}' resources: gpu: H200 + disk: 200GB.. port: 8000 model: zai-org/GLM-4.5-Air-FP8 - -# Custom probe is required for PD disaggregation. -probes: - - type: http - url: /health - interval: 15s ```
-> With the the `dynamo` router, you can use SGLang, vLLM, and TensorRT-LLM prefill and decode workers. +The service consists of four replica groups: + +- `etcd` runs [etcd](https://etcd.io/), which Dynamo uses for service discovery. Other groups reach it via + [`${{ groups[0].replicas[0].IP_ADDRESS }}`](../../concepts/services.md#replica-groups). +- `frontend` runs the Dynamo frontend, which serves the OpenAI-compatible API and routes requests to the workers + based on KV cache overlap. Because the group sets `router`, `dstack` sends the service traffic to it. +- `prefill-worker` and `decode-worker` run vLLM. The prefill workers pass the KV cache to the decode workers over NIXL. + +> With the `dynamo` router, you can use SGLang, vLLM, and TensorRT-LLM prefill and decode workers. Save the configuration as `service.dstack.yml`, then use the [`dstack apply`](../../reference/cli/dstack/apply.md) command. @@ -158,9 +159,11 @@ Currently, auto-scaling only supports `rps` as the metric. TTFT and ITL metrics !!! info "Cluster" PD disaggregation requires the service to run in a fleet with `placement` set to `cluster`, because the replicas require an interconnect between instances. - While the prefill and decode replicas run on GPUs, the router replica requires a CPU instance in the same cluster. + While the prefill and decode replicas run on GPUs, the `etcd` and `frontend` replicas require CPU instances in the same cluster. + + For production, use instances with RDMA (InfiniBand or RoCE). Without RDMA, NIXL transfers the KV cache over TCP, which works but may be slower than recomputing the prompt on the decode worker. ## What's next? 1. Read about [services](../../concepts/services.md) and [gateways](../../concepts/gateways.md) -2. Browse the [NVIDIA Dynamo GitHub repository](https://github.com/ai-dynamo/dynamo) and the [SGLang](./sglang.md) example +2. Browse the [NVIDIA Dynamo GitHub repository](https://github.com/ai-dynamo/dynamo) and the [vLLM](./vllm.md) example diff --git a/src/dstack/_internal/core/models/routers.py b/src/dstack/_internal/core/models/routers.py index 647c62b2c..c97bd27cc 100644 --- a/src/dstack/_internal/core/models/routers.py +++ b/src/dstack/_internal/core/models/routers.py @@ -24,7 +24,7 @@ class ReplicaGroupRouterConfig(CoreModel): "The router implementation for this replica group." " `smg` runs the Shepherd Model Gateway (SMG) and dstack syncs worker URLs to it." " `dynamo` runs the NVIDIA Dynamo frontend, which discovers workers" - " itself via etcd/NATS" + " itself via etcd" ), ), ] = RouterType.SMG diff --git a/src/dstack/_internal/server/services/jobs/configurators/base.py b/src/dstack/_internal/server/services/jobs/configurators/base.py index 425961915..2d18ba6ed 100644 --- a/src/dstack/_internal/server/services/jobs/configurators/base.py +++ b/src/dstack/_internal/server/services/jobs/configurators/base.py @@ -531,7 +531,7 @@ def _probes(self) -> list[ProbeSpec]: # The router answers chat completions only once it has workers, which it gets # regardless of its own readiness. For SMG routers, dstack registers workers # via `ServiceRouterWorkerSyncWorker`, which doesn't check router readiness. - # Dynamo workers register themselves with the router via etcd/NATS. + # Dynamo workers register themselves in etcd, which the Dynamo frontend watches. return [_openai_model_probe_spec(model.name, model.prefix)]