diff --git a/docs/en/deploy/inference_service/guides/assets/llm-d-multi-node-vllm-2x4.yaml b/docs/en/deploy/inference_service/guides/assets/llm-d-multi-node-vllm-2x4.yaml new file mode 100644 index 0000000..9ee3bd2 --- /dev/null +++ b/docs/en/deploy/inference_service/guides/assets/llm-d-multi-node-vllm-2x4.yaml @@ -0,0 +1,191 @@ +# Replace all example values before applying this manifest. +# Topology: one logical service, two nodes, four GPUs per Pod, TP=4, PP=2. +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceServiceConfig +metadata: + name: kserve-config-llm-worker-pipeline-parallel + namespace: your-namespace +spec: {} +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceService +metadata: + name: vllm-multinode + namespace: your-namespace +spec: + model: + name: distributed-model + uri: pvc://your-model-pvc/path/to/model + replicas: 1 + parallelism: + tensor: 4 + pipeline: 2 + router: + gateway: {} + route: {} + scheduler: {} + template: + terminationGracePeriodSeconds: 60 + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + leaderworkerset.sigs.k8s.io/name: vllm-multinode-kserve-mn + topologyKey: kubernetes.io/hostname + containers: + - name: main + image: your-registry/vllm-openai:your-tag + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -lc + args: + - | + set -euo pipefail + + : "${VLLM_HOST_IP:?VLLM_HOST_IP was not injected}" + + exec vllm serve /mnt/models \ + --host 0.0.0.0 \ + --port 8000 \ + --served-model-name distributed-model \ + --distributed-executor-backend mp \ + --tensor-parallel-size 4 \ + --pipeline-parallel-size 2 \ + --nnodes 2 \ + --node-rank 0 \ + --master-addr "${VLLM_HOST_IP}" \ + --master-port 29501 \ + --gpu-memory-utilization 0.90 \ + --max-model-len 131072 \ + --disable-uvicorn-access-log + env: + - name: VLLM_HOST_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: NCCL_DEBUG + value: INFO + - name: NCCL_ASYNC_ERROR_HANDLING + value: "1" + ports: + - name: http + containerPort: 8000 + protocol: TCP + - name: dist-init + containerPort: 29501 + protocol: TCP + resources: + requests: + cpu: "16" + memory: 128Gi + nvidia.com/gpu: "4" + limits: + cpu: "16" + memory: 128Gi + nvidia.com/gpu: "4" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - IPC_LOCK + drop: + - ALL + seccompProfile: + type: RuntimeDefault + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 360 + readinessProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 30 + timeoutSeconds: 5 + failureThreshold: 5 + volumeMounts: + - name: dshm + mountPath: /dev/shm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 32Gi + worker: + terminationGracePeriodSeconds: 60 + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + leaderworkerset.sigs.k8s.io/name: vllm-multinode-kserve-mn + topologyKey: kubernetes.io/hostname + containers: + - name: main + image: your-registry/vllm-openai:your-tag + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - -lc + args: + - | + set -euo pipefail + + : "${LWS_LEADER_ADDRESS:?LWS_LEADER_ADDRESS was not injected}" + + exec vllm serve /mnt/models \ + --served-model-name distributed-model \ + --distributed-executor-backend mp \ + --tensor-parallel-size 4 \ + --pipeline-parallel-size 2 \ + --nnodes 2 \ + --node-rank 1 \ + --master-addr "${LWS_LEADER_ADDRESS}" \ + --master-port 29501 \ + --gpu-memory-utilization 0.90 \ + --max-model-len 131072 \ + --disable-uvicorn-access-log \ + --headless + env: + - name: NCCL_DEBUG + value: INFO + - name: NCCL_ASYNC_ERROR_HANDLING + value: "1" + resources: + requests: + cpu: "16" + memory: 128Gi + nvidia.com/gpu: "4" + limits: + cpu: "16" + memory: 128Gi + nvidia.com/gpu: "4" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - IPC_LOCK + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: dshm + mountPath: /dev/shm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 32Gi diff --git a/docs/en/deploy/inference_service/guides/llm-d-multi-node.mdx b/docs/en/deploy/inference_service/guides/llm-d-multi-node.mdx new file mode 100644 index 0000000..057bfbf --- /dev/null +++ b/docs/en/deploy/inference_service/guides/llm-d-multi-node.mdx @@ -0,0 +1,261 @@ +--- +weight: 105 +i18n: + title: + en: Deploy a Multi-Node vLLM Service with llm-d + zh: 使用 llm-d 跨节点部署 vLLM 服务 +--- + +# Deploy a Multi-Node vLLM Service with llm-d + +## Overview + +Use this pattern when one model instance needs GPUs from more than one Kubernetes +node. The example creates one logical vLLM service across two nodes with four GPUs +per node: + +```text +LLMInferenceService replicas=1 +└── LeaderWorkerSet group + ├── leader: 4 GPUs, node rank 0, OpenAI-compatible API + └── worker: 4 GPUs, node rank 1, headless + +tensor parallel = 4 +pipeline parallel = 2 +total GPUs = 4 x 2 = 8 +``` + +The components have separate responsibilities: + +- `LLMInferenceService` describes the logical service and its leader and worker + Pod specifications. +- LeaderWorkerSet (LWS) creates and manages the leader and worker as one group. +- vLLM multiprocessing and NCCL execute tensor and pipeline parallelism across + the GPUs. +- llm-d routing components, including the inference scheduler/EPP, route requests + to available backend groups. They do not implement model parallelism or combine + GPUs by themselves. + +:::warning +Do not replace this topology with a single Pod that requests eight GPUs. Kubernetes +and GPU schedulers allocate a Pod on one node; free GPUs on different nodes cannot +be combined into one Pod. Also, `replicas: 2` creates two independent +leader/worker groups. It does not turn two four-GPU replicas into one eight-GPU +model instance. +::: + +## Prerequisites + +Before you deploy, confirm all of the following: + +- Alauda Build of KServe and Alauda Build of LeaderWorkerSet are installed. +- The cluster has two schedulable GPU nodes with at least four allocatable GPUs + each. +- Both nodes run compatible GPU drivers, CUDA, NCCL, and RDMA software. +- The runtime image supports the model, GPU architecture, quantization format, + attention backend, and the vLLM multi-node command-line arguments used here. +- Both Pods can read identical model files. Use an RWX PVC, an OCI ModelCar that + can be pulled by both nodes, or separate node-local copies with identical + contents. +- Network policy and host networking allow TCPStore rendezvous on port `29501` + and the ports selected dynamically by NCCL. Low-performance Ethernet can make + the service functional but unsuitable for production. +- CPU and memory requests fit on both GPU nodes. GPU capacity alone is not enough + for scheduling. + +Check that the required APIs are installed: + +```bash +kubectl api-resources | grep -E 'LLMInferenceService|LeaderWorkerSet' +``` + +## Prepare the Manifest + +Download or copy the +[two-node, four-GPU-per-node template](https://github.com/alauda/aml-docs/tree/master/docs/en/deploy/inference_service/guides/assets/llm-d-multi-node-vllm-2x4.yaml). + +Replace these values before applying it: + +| Placeholder | Replace with | +| --- | --- | +| `your-namespace` | Namespace where the service and model PVC exist | +| `your-registry/vllm-openai:your-tag` | Runtime image validated for the model and GPU architecture; use an immutable digest in production | +| `pvc://your-model-pvc/path/to/model` | Shared model URI, or the storage URI used by your cluster | +| `distributed-model` | Name exposed through the OpenAI-compatible API | +| `--max-model-len 131072` | Context limit that fits the available KV cache and expected concurrency | +| `nvidia.com/gpu` | GPU resource key used by the cluster | + +The template uses the standard `nvidia.com/gpu` resource. If the cluster uses +HAMi, replace it with the HAMi resource contract used by an existing working Pod, +for example `nvidia.com/gpualloc`, `nvidia.com/gpucores`, and +`nvidia.com/gpumem`. HAMi still evaluates each Pod independently on one node. + +Add model-specific flags only after checking the exact runtime image. Examples +include quantization, Expert Parallel, tool-call parser, reasoning parser, and a +hardware-specific attention backend. Keep speculative decoding disabled for the +first bring-up so it is not mixed with multi-node troubleshooting. + +### Understand the Parallelism Fields + +The Kubernetes and vLLM settings must agree: + +| Layer | Setting | Example value | +| --- | --- | ---: | +| Logical service | `spec.replicas` | 1 | +| LLMInferenceService | `spec.parallelism.tensor` | 4 | +| LLMInferenceService | `spec.parallelism.pipeline` | 2 | +| Leader vLLM | `--tensor-parallel-size` | 4 | +| Leader vLLM | `--pipeline-parallel-size` | 2 | +| Worker vLLM | `--tensor-parallel-size` | 4 | +| Worker vLLM | `--pipeline-parallel-size` | 2 | +| Both vLLM Pods | `--nnodes` | 2 | + +The leader uses its Pod IP as `--master-addr`. LWS injects +`LWS_LEADER_ADDRESS` into the worker, which uses that address to join the same +TCPStore and NCCL process group. The worker runs with `--headless`; only the +leader exposes the inference API. + +## Apply the Service + +Apply the manifest: + +```bash +kubectl apply -f llm-d-multi-node-vllm-2x4.yaml +``` + +You do not need to delete an existing `LLMInferenceService` before applying an +updated Pod template. KServe and LWS reconcile the generated workload. + +Watch both the logical service and its Pods: + +```bash +watch -n 5 'kubectl -n your-namespace get llminferenceservice vllm-multinode; kubectl -n your-namespace get pods -o wide | grep -E "NAME|vllm-multinode"' +``` + +The expected result is one leader Pod and one worker Pod on different nodes. The +leader becomes Ready only after both nodes join the distributed engine and the +OpenAI-compatible API starts. + +## Verify with a Real Inference Request + +`Ready=True` and `/health` are necessary but do not prove that a request can pass +through every pipeline stage. Port-forward the leader Pod: + +```bash +kubectl -n your-namespace port-forward pod/vllm-multinode-kserve-mn-0 8000:8000 +``` + +In another terminal, send a real request: + +```bash +curl -sS http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{"model":"distributed-model","stream":false,"messages":[{"role":"user","content":"Reply with: multi-node inference succeeded"}],"temperature":0,"max_tokens":64}' +``` + +An HTTP 200 response with generated content confirms a basic end-to-end request. +Before production, also verify representative long contexts, concurrent requests, +multi-turn conversations, tool calls, Pod restarts, and latency and throughput. + +## Validated Example: GLM-5.2 on A100 + +This deployment pattern was validated on AML 2.7 with +[QuantTrio/GLM-5.2-Int4-Int8Mix](https://huggingface.co/QuantTrio/GLM-5.2-Int4-Int8Mix): + +- two nodes with four NVIDIA A100 80 GB GPUs per node; +- tensor parallel size 4 and pipeline parallel size 2; +- Expert Parallel enabled and speculative decoding disabled; +- one real `/v1/chat/completions` request completed successfully with HTTP 200. + +The deployment used a customer-internal vLLM image built from the SM80 sparse +MLA implementation proposed in +[vLLM PR #47629](https://github.com/vllm-project/vllm/pull/47629). With one +backend group, this result validates LLMInferenceService, LWS, vLLM, and NCCL +multi-node execution. It does not benchmark llm-d scheduling or routing. + +:::warning +As of August 10, 2026, PR #47629 has not been merged into an official vLLM +release. Public vLLM images do not provide the SM80 sparse MLA backend required +by GLM-5.2 on A100. Quantizing the model reduces weight memory but does not +remove this runtime requirement. The model card's published vLLM verification +uses eight H200 GPUs, not A100 GPUs. + +The customer image is not publicly available. Other A100 deployments must build +and validate their own image from a pinned PR commit. This result validates basic +multi-node inference only; it does not constitute long-context, concurrency, +performance, tool-calling, or production-stability validation. +::: + +## Troubleshooting + +### LLMInferenceServiceConfig Is Missing + +Some KServe installations look up a preset when both `spec.worker` and +`spec.parallelism.pipeline` are present. A reconciliation event similar to the +following means the referenced preset does not exist: + +```text +failed to get LLMInferenceServiceConfig +"kserve-config-llm-worker-pipeline-parallel" not found +``` + +Create the preset with the exact name and API version installed in the cluster. +The supplied template includes an empty namespace-local overlay because it +provides complete leader and worker Pod specifications. Do not guess the API +version; verify it first: + +```bash +kubectl get crd llminferenceserviceconfigs.serving.kserve.io -o jsonpath='{.spec.versions[*].name}' +``` + +For AML releases that serve `v1alpha2`, use +`serving.kserve.io/v1alpha2`. A `no matches for kind` error indicates that the +manifest uses an API version not served by that cluster. + +### The Worker Pod Is Pending + +Inspect the scheduler event instead of relying only on `nvidia-smi`: + +```bash +kubectl -n your-namespace describe pod | tail -n 30 +``` + +Common causes include: + +- insufficient allocatable GPU count or GPU memory according to HAMi; +- insufficient CPU or system memory; +- node selectors, taints, affinity, or topology rules; +- a model PVC that cannot be mounted on the selected node. + +Physical free memory shown by `nvidia-smi` is not the same as scheduler-visible +capacity. Check resource requests, HAMi allocation annotations, and all Pods that +already reserve GPUs on the node. + +### Ranks Join and Then Report TCPStore or NCCL Broken Pipe + +TCPStore `Broken pipe`, NCCL heartbeat warnings, and process termination at the +end of the log are often secondary errors after one rank has already exited. +Find the earliest engine or worker exception in both Pods: + +```bash +kubectl -n your-namespace logs -p --all-containers --prefix | grep -m1 -A40 -E 'Traceback|ValueError|RuntimeError|ERROR' +``` + +Typical primary causes are an unsupported attention backend, a runtime image that +does not support the GPU architecture, model-loading failure, or insufficient KV +cache for the model's default maximum sequence length. + +### The Default Context Length Does Not Fit + +vLLM checks whether at least one request at `max_model_len` can fit in the +available KV cache. If startup reports the required and available KV cache sizes, +set `--max-model-len` below the estimated maximum and retain operational margin +for concurrency and memory variation. Increasing `--gpu-memory-utilization` can +help only when the additional GPU memory is genuinely available; it does not fix +an incompatible kernel or runtime. + +## Related Documentation + +- [Alauda Build of LeaderWorkerSet](../../components/lws/intro.mdx) +- [Alauda Build of KServe](../../components/kserve/intro.mdx) +- [Enable Expert Parallel for vLLM Inference Services](./vllm_expert_parallel.mdx) +- [llm-d](https://github.com/llm-d/llm-d) +- [vLLM Parallelism and Scaling](https://docs.vllm.ai/en/latest/serving/parallelism_scaling/) diff --git a/docs/en/deploy/inference_service/guides/vllm_expert_parallel.mdx b/docs/en/deploy/inference_service/guides/vllm_expert_parallel.mdx index 0e436f3..1b54336 100644 --- a/docs/en/deploy/inference_service/guides/vllm_expert_parallel.mdx +++ b/docs/en/deploy/inference_service/guides/vllm_expert_parallel.mdx @@ -209,7 +209,11 @@ This review confirms that the intended vLLM arguments were applied to the servic Multi-node EP deployments require additional distributed runtime and networking configuration, including per-node launch settings, node roles, and data-parallel communication settings. :::warning -This page focuses on the single-node configuration pattern. If you need multi-node EP, refer to the official vLLM guide and adapt the deployment model to your cluster topology and runtime environment. +This page focuses on the single-node configuration pattern. For the Alauda AI +leader/worker deployment workflow, see +[Deploy a Multi-Node vLLM Service with llm-d](./llm-d-multi-node.mdx), then add +the EP flags supported by your model and runtime image. Also review the official +vLLM guide for the EP-specific communication topology. ::: ## References \{#references} diff --git a/docs/en/deploy/inference_service/inference_service.mdx b/docs/en/deploy/inference_service/inference_service.mdx index 5394890..b26a42e 100644 --- a/docs/en/deploy/inference_service/inference_service.mdx +++ b/docs/en/deploy/inference_service/inference_service.mdx @@ -110,7 +110,7 @@ AML completes the release and operation of cloud native inference services based | Model Type | Required when Model Location is `PVC`. Choose between `Generative AI model` or `Predictive Model`. This selection determines the available Runtimes. | | Model | Required when Model Location is `Model repository`. The name of the model used for inference. | | Version | Required when Model Location is `Model repository`. The version of the model. Options include Branch and Tag. | -| Inference Runtimes | Required, The engine used for inference runtime. When `Model Type` is `Generative AI model`, `llm-d` is available for distributed inference. | +| Inference Runtimes | Required, The engine used for inference runtime. When `Model Type` is `Generative AI model`, `llm-d` is available for distributed inference. For one model instance that spans multiple nodes, see [Deploy a Multi-Node vLLM Service with llm-d](./guides/llm-d-multi-node.mdx). | | Config Source | Required, Choose the source for deploying resources: `Hardware profile` or `Custom`. | | Hardware Profile | Required when Config Source is `Hardware profile`. Select a predefined hardware profile that configures the required resources. | | Requests CPU | Required when Config Source is `Custom`. The amount of CPU resources requested by the inference service. |