diff --git a/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml b/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml new file mode 100644 index 00000000..115de060 --- /dev/null +++ b/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml @@ -0,0 +1,451 @@ +# ============================================================================= +# google/gemma-4-31B-it · P4 + D4 · KServe LLMInferenceService + InferNex Bridge +# +# Standard P/D delivery shape derived from the already validated MiniMax P8+D8 +# manifest. InferNex Bridge owns the P/D proxy; KServe owns Prefill/Decode, +# InferencePool, scheduler, route, and lifecycle resources. +# +# Decode: 100.1.69.8 · TP4 · FULL_DECODE_ONLY · Mooncake kv_consumer +# Prefill: 100.1.69.9 · TP4 · PIECEWISE ACLGraph · Mooncake kv_producer +# ============================================================================= +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceServiceConfig +metadata: + name: kserve-config-llm-decode-template + namespace: zgsu-ns1 + labels: + app.kubernetes.io/part-of: infernex-bridge +spec: + template: + initContainers: [] +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceServiceConfig +metadata: + name: gemma4-31b-it-pd-engine + namespace: kserve + labels: + app.kubernetes.io/name: gemma4-31b-it-pd-engine + app.kubernetes.io/part-of: infernex-bridge +spec: + template: + initContainers: [] + nodeSelector: + kubernetes.io/hostname: 100.1.69.8 + hostIPC: true + tolerations: + - operator: Exists + terminationGracePeriodSeconds: 60 + containers: + - name: main + image: quay.io/ascend/vllm-ascend@sha256:fa246685a9e0b9ea63f242be634dcc7616060f6f7dc9ca7018fa06c1657940da + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + set +u + source /usr/local/Ascend/ascend-toolkit/set_env.sh 2>/dev/null || true + source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true + set -eo pipefail + printf '{}' > /app/mooncake.json + # AscendDirect may allocate auxiliary transport ports in 20000-27999. + # Keep the two application KV listeners above that range so a P/D + # restart cannot collide with a transport listener. + KV_CONFIG=$(printf '{"engine_id":"%s","kv_buffer_device":"npu","kv_connector":"MooncakeConnectorV1","kv_connector_extra_config":{"decode":{"dp_size":1,"tp_size":4},"prefill":{"dp_size":1,"tp_size":4},"use_ascend_direct":true},"kv_parallel_size":1,"kv_port":"28001","kv_rank":1,"kv_role":"kv_consumer"}' "$POD_NAME") + echo "[gemma4-pd-decode] pod=$POD_NAME ip=$POD_IP devices=$(ls /dev/davinci* 2>/dev/null | tr '\n' ' ')" + exec vllm serve /mnt/models \ + --host 0.0.0.0 \ + --port 8000 \ + --served-model-name "{{ .Spec.Model.Name }}" \ + --seed 1024 \ + --trust-remote-code \ + --api-server-count 1 \ + --enable-auto-tool-choice \ + --tool-call-parser gemma4 \ + --reasoning-parser gemma4 \ + --tensor-parallel-size 4 \ + --max-model-len -1 \ + --max-num-batched-tokens 128 \ + --max-num-seqs 96 \ + --block-size 128 \ + --gpu-memory-utilization 0.9 \ + --no-enable-prefix-caching \ + --async-scheduling \ + --compilation-config '{"cudagraph_mode":"FULL_DECODE_ONLY"}' \ + --kv-transfer-config "$KV_CONFIG" + env: &engine-env + - { name: HOME, value: /home } + - { name: VLLM_LOGGING_LEVEL, value: INFO } + - { name: HF_HUB_OFFLINE, value: "1" } + - { name: TRANSFORMERS_OFFLINE, value: "1" } + - { name: PYTHONUNBUFFERED, value: "1" } + - { name: PYTHONHASHSEED, value: "0" } + - { name: VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES, value: "1" } + - { name: MOONCAKE_CONFIG_PATH, value: /app/mooncake.json } + - { name: PYTORCH_NPU_ALLOC_CONF, value: "expandable_segments:True" } + - { name: HCCL_OP_EXPANSION_MODE, value: AIV } + - { name: HCCL_BUFFSIZE, value: "1024" } + - { name: HCCL_INTRA_PCIE_ENABLE, value: "0" } + # Current AscendDirect on Atlas A2/910B3 requires the HCCN RoCE + # device IPs for cross-node P/D KV transfer. + - { name: HCCL_INTRA_ROCE_ENABLE, value: "1" } + - { name: OMP_PROC_BIND, value: "false" } + - { name: OMP_NUM_THREADS, value: "1" } + - { name: TASK_QUEUE_ENABLE, value: "1" } + - { name: VLLM_ENGINE_READY_TIMEOUT_S, value: "3000" } + - { name: VLLM_RPC_TIMEOUT, value: "6000" } + - { name: VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS, value: "3000" } + - { name: HCCL_EXEC_TIMEOUT, value: "600" } + - { name: HCCL_CONNECT_TIMEOUT, value: "300" } + # Keep per-operation HCCL entries while diagnosing cross-node + # AscendDirect KV transfer failures. Remove after root cause is known. + - { name: HCCL_ENTRY_LOG_ENABLE, value: "1" } + - name: POD_NAME + valueFrom: + fieldRef: { fieldPath: metadata.name } + - name: POD_IP + valueFrom: + fieldRef: { fieldPath: status.podIP } + - name: HCCL_IF_IP + valueFrom: + fieldRef: { fieldPath: status.podIP } + - { name: GLOO_SOCKET_IFNAME, value: eth0 } + - { name: TP_SOCKET_IFNAME, value: eth0 } + - { name: HCCL_SOCKET_IFNAME, value: eth0 } + ports: + - { name: http, containerPort: 8000, protocol: TCP } + - { name: kv, containerPort: 28001, protocol: TCP } + resources: + requests: + cpu: "16" + memory: 96Gi + huawei.com/Ascend910: "4" + limits: + cpu: "48" + memory: 256Gi + huawei.com/Ascend910: "4" + securityContext: &engine-security-context + allowPrivilegeEscalation: false + capabilities: { drop: ["ALL"] } + runAsGroup: 0 + runAsNonRoot: false + runAsUser: 0 + seccompProfile: { type: RuntimeDefault } + startupProbe: + httpGet: { path: /health, port: http } + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 160 + readinessProbe: + httpGet: { path: /health, port: http } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: { path: /health, port: http } + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 8 + volumeMounts: + - { name: model, mountPath: /mnt/models, readOnly: true } + - { name: shm, mountPath: /dev/shm } + - { name: mooncake-config, mountPath: /app } + # AscendDirect on Atlas A2 reads each NPU's RoCE device IP from + # the node-local HCCN configuration when building its ranktable. + - { name: hccn-config, mountPath: /etc/hccn.conf, readOnly: true } + volumes: + - name: model + persistentVolumeClaim: { claimName: gemma4-31b-n8 } + - name: shm + emptyDir: { medium: Memory, sizeLimit: 32Gi } + - name: mooncake-config + emptyDir: {} + - name: hccn-config + hostPath: { path: /etc/hccn.conf, type: File } + + prefill: + replicas: 1 + template: + nodeSelector: + kubernetes.io/hostname: 100.1.69.9 + hostIPC: true + tolerations: + - operator: Exists + terminationGracePeriodSeconds: 60 + containers: + - name: main + image: quay.io/ascend/vllm-ascend@sha256:fa246685a9e0b9ea63f242be634dcc7616060f6f7dc9ca7018fa06c1657940da + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + set +u + source /usr/local/Ascend/ascend-toolkit/set_env.sh 2>/dev/null || true + source /usr/local/Ascend/nnal/atb/set_env.sh 2>/dev/null || true + set -eo pipefail + printf '{}' > /app/mooncake.json + KV_CONFIG=$(printf '{"engine_id":"%s","kv_buffer_device":"npu","kv_connector":"MooncakeConnectorV1","kv_connector_extra_config":{"decode":{"dp_size":1,"tp_size":4},"prefill":{"dp_size":1,"tp_size":4},"use_ascend_direct":true},"kv_parallel_size":1,"kv_port":"28000","kv_rank":0,"kv_role":"kv_producer"}' "$POD_NAME") + echo "[gemma4-pd-prefill] pod=$POD_NAME ip=$POD_IP devices=$(ls /dev/davinci* 2>/dev/null | tr '\n' ' ')" + exec vllm serve /mnt/models \ + --host 0.0.0.0 \ + --port 8000 \ + --served-model-name "{{ .Spec.Model.Name }}" \ + --seed 1024 \ + --trust-remote-code \ + --api-server-count 1 \ + --enable-auto-tool-choice \ + --tool-call-parser gemma4 \ + --reasoning-parser gemma4 \ + --tensor-parallel-size 4 \ + --max-model-len -1 \ + --max-num-batched-tokens 8192 \ + --max-num-seqs 96 \ + --block-size 128 \ + --gpu-memory-utilization 0.9 \ + --no-enable-prefix-caching \ + --compilation-config '{"cudagraph_mode":"PIECEWISE"}' \ + --kv-transfer-config "$KV_CONFIG" + env: *engine-env + ports: + - { name: http, containerPort: 8000, protocol: TCP } + - { name: kv, containerPort: 28000, protocol: TCP } + resources: + requests: + cpu: "16" + memory: 96Gi + huawei.com/Ascend910: "4" + limits: + cpu: "48" + memory: 256Gi + huawei.com/Ascend910: "4" + securityContext: *engine-security-context + startupProbe: + httpGet: { path: /health, port: http } + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 160 + readinessProbe: + httpGet: { path: /health, port: http } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + livenessProbe: + httpGet: { path: /health, port: http } + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 8 + volumeMounts: + - { name: model, mountPath: /mnt/models, readOnly: true } + - { name: shm, mountPath: /dev/shm } + - { name: mooncake-config, mountPath: /app } + - { name: hccn-config, mountPath: /etc/hccn.conf, readOnly: true } + volumes: + - name: model + persistentVolumeClaim: { claimName: gemma4-31b-n9 } + - name: shm + emptyDir: { medium: Memory, sizeLimit: 32Gi } + - name: mooncake-config + emptyDir: {} + - name: hccn-config + hostPath: { path: /etc/hccn.conf, type: File } +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceServiceConfig +metadata: + name: gemma4-31b-it-pd-scheduler + namespace: kserve + labels: + app.kubernetes.io/name: gemma4-31b-it-pd-scheduler + app.kubernetes.io/part-of: infernex-bridge +spec: + router: + scheduler: + pool: + spec: + endpointPickerRef: + failureMode: FailOpen + group: "" + kind: Service + name: "{{ ChildName .ObjectMeta.Name `-epp-service` }}" + port: { number: 9002 } + selector: + matchLabels: + app.kubernetes.io/name: "{{ .ObjectMeta.Name }}" + app.kubernetes.io/part-of: llminferenceservice + kserve.io/component: workload + targetPorts: + - number: 8000 + template: + nodeSelector: + kubernetes.io/hostname: 100.1.69.8 + tolerations: + - operator: Exists + serviceAccountName: "{{ ChildName .ObjectMeta.Name `-epp-sa` }}" + containers: + - name: main + image: 100.1.69.8:11443/3rdparty/openfuyao/hermes-router:26.6.0-rc.3 + imagePullPolicy: IfNotPresent + args: + - --pool-name + - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}" + - --pool-namespace + - "{{ .ObjectMeta.Namespace }}" + - --pool-group + - inference.networking.k8s.io + - --zap-encoder + - json + - --v + - "1" + - --tracing=false + - --metrics-endpoint-auth=false + # A single fixed P/D pair does not need sub-second load updates. + # Keep routing metrics useful without adding two /metrics scrapes + # per engine every second (direct plus proxy aggregation). + - --refresh-metrics-interval=15s + - --config-text + - |- + # This is Hermes Router's embedded config-document schema, not + # a Kubernetes resource API group. InferNex 26.6.0-rc.3 still + # requires the x-k8s version here; the actual InferencePool and + # --pool-group above both use inference.networking.k8s.io. + apiVersion: inference.networking.x-k8s.io/v1alpha1 + kind: EndpointPickerConfig + # One fixed Prefill/Decode pair has no prefix-routing choice. + # Keep tokenization out of the scheduler path; the 26.6.0-rc.3 + # tokenizer RPC has a 5s init deadline while Gemma needs >17s. + plugins: + - type: metrics-data-source + parameters: { scheme: "http", path: "/metrics", insecureSkipVerify: true } + - type: core-metrics-extractor + - type: pd-group-producer + parameters: + pdLabelName: app.kubernetes.io/component + pdGroupLabelName: app.kubernetes.io/name + prefillValue: llminferenceservice-workload-prefill + decodeValue: llminferenceservice-workload + leaderValue: pd-proxy + - type: request-lifecycle-tracker + parameters: + storeName: gemma4-pd-routing + persistence: { enabled: false } + - type: inflight-data-producer + parameters: { storeName: gemma4-pd-routing } + - type: pd-group-filter + - type: pd-kvcache-aware-scorer + parameters: + prefillKVUsageWeight: 1 + prefillQueueWeight: 1 + prefillInflightWeight: 1 + decodeKVUsageWeight: 1 + decodeQueueWeight: 1 + decodeInflightWeight: 1 + prefillScoreWeight: 1 + decodeScoreWeight: 1 + - type: min-score-picker + - type: pd-header-handler + schedulingProfiles: + - name: default + plugins: + - pluginRef: pd-group-producer + - pluginRef: request-lifecycle-tracker + - pluginRef: inflight-data-producer + - pluginRef: pd-group-filter + - pluginRef: pd-kvcache-aware-scorer + - pluginRef: min-score-picker + - pluginRef: pd-header-handler + ports: + - { containerPort: 9002, name: grpc, protocol: TCP } + - { containerPort: 9003, name: grpc-health, protocol: TCP } + - { containerPort: 9090, name: metrics, protocol: TCP } + startupProbe: + grpc: { port: 9003, service: liveness } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 30 + livenessProbe: + grpc: { port: 9003, service: liveness } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + grpc: { port: 9003, service: readiness } + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + resources: + requests: { cpu: "1", memory: 1Gi } + limits: { memory: 4Gi } + securityContext: + allowPrivilegeEscalation: false + capabilities: { drop: ["ALL"] } + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: { type: RuntimeDefault } + volumeMounts: + - { name: tls-certs, mountPath: /etc/ssl/certs, readOnly: true } + volumes: + - name: tls-certs + secret: + secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}" + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: serving.kserve.io/v1alpha2 +kind: LLMInferenceService +metadata: + name: gemma4-31b-it-pd + namespace: zgsu-ns1 + labels: + infernex.io/runtime: "true" + annotations: + infernex.io/disabled-components: "mooncake,cache-indexer,pd-orchestrator,eagle-eye" + model-auto.alauda.io/config-revision: "0803-p4d4-main-pc-piecewise-hccn-no-tokenizer-hccldiag" + serving.kserve.io/stop: "false" +spec: + model: + name: google/gemma-4-31B-it + uri: pvc://gemma4-31b-n8/ + storageInitializer: + enabled: false + baseRefs: + - name: gemma4-31b-it-pd-engine + - name: gemma4-31b-it-pd-scheduler + router: + gateway: {} + route: + http: + spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: kserve-ingress-gateway + namespace: kserve + rules: + - matches: + - path: + type: PathPrefix + value: /zgsu-ns1/gemma4-31b-it-pd + filters: + - type: URLRewrite + urlRewrite: + path: + type: ReplacePrefixMatch + replacePrefixMatch: / + backendRefs: + - group: "" + kind: Service + name: gemma4-31b-it-pd-proxy-server + port: 8000 + weight: 1 + timeouts: + request: 0s + backendRequest: 0s + replicas: 1 + template: + enableServiceLinks: false + prefill: + replicas: 1 + template: + enableServiceLinks: false diff --git a/docs/en/plan/validated_models/gemma4-31b-it.mdx b/docs/en/plan/validated_models/gemma4-31b-it.mdx new file mode 100644 index 00000000..59da3bc6 --- /dev/null +++ b/docs/en/plan/validated_models/gemma4-31b-it.mdx @@ -0,0 +1,172 @@ +--- +weight: 50 +i18n: + title: + en: Gemma-4-31B-it + zh: Gemma-4-31B-it +--- + +# Gemma-4-31B-it + +`Gemma4ForConditionalGeneration` is a dense multimodal model. This validation covers +its text-generation serving path: the text backbone has 60 layers, hidden size 5,376, +32 attention heads / 16 KV heads, a 262,144-token vocabulary, and a native 262,144-token +context window. The validated deployment is **P4+D4**: one 4-card Prefill engine and one +4-card Decode engine, coordinated by KServe `LLMInferenceService` and the InferNex +Bridge. It is an **8 × Ascend 910B3 64 GB** composite service, not a 4-card replica. + +## Model identity + +| Field | Value | +|---|---| +| Publisher / model | Google / `google/gemma-4-31B-it` | +| Architecture | `gemma4` / `Gemma4ForConditionalGeneration`; dense multimodal model, validated here for text generation | +| Text backbone | 60 layers; hidden size 5,376; 32 attention heads / 16 KV heads | +| Context / vocabulary | 262,144 / 262,144 | +| Precision | BF16 | + +## Validated hardware × stack + +| Platform | Engine | Version / configuration | Status | +|---|---|---|---| +| Ascend 910B3 64 GB × 8 across 2 nodes | vLLM-Ascend | `nightly-main-openeuler-20260804-b481d79c` (ARM64 manifest `sha256:fa246685...`, CANN 9.0.1) | P4+D4 closed-loop AIPerf validation through MaaS | + +:::note +This is a dated `nightly-main` snapshot built from vLLM-Ascend commit +`b481d79c7de49ab285e61cec689172931c124429`. The manifest pins its public ARM64 digest; +it is the exact image content used by the validated deployment, not a claim that the tag +is the newest vLLM-Ascend build. +::: + +## Model configuration + +| Component | Setting | +|---|---| +| Prefill | TP=4 on one 910B3 node; `max-num-batched-tokens=8192`; `PIECEWISE` ACLGraph | +| Decode | TP=4 on the second node; `max-num-batched-tokens=128`; `FULL_DECODE_ONLY` graph capture | +| P/D handoff | `MooncakeConnectorV1`, Ascend Direct KV transfer, TP4/TP4 contract | +| Capacity guardrail | `max-num-seqs=96` on both engines; `gpu-memory-utilization=0.9` | +| Prefix cache | disabled in the engine; the workload uses explicit P/D KV transfer | +| Scheduler | one fixed P/D pair; metrics-based P/D scoring without a tokenizer sidecar | +| Tool calls and reasoning | `--enable-auto-tool-choice --tool-call-parser gemma4 --reasoning-parser gemma4` | + +## Why P4+D4 + +The earlier aggregated TP4 deployment and P4+D4 are different resource envelopes. The +aggregate service uses four NPUs and mixes long Prefill work with Decode; P4+D4 uses eight +NPUs and removes Prefill dynamic steps from the Decode process. In the earlier capacity +study, aggregate TP4 reached only 2.554 completed RPS on the 512/256 chatbot workload at +target rate 3 over 900 seconds, with `Running=104` and `Waiting=294` at the end. P4+D4 is +therefore the production candidate, but its throughput must never be presented as a +four-card aggregate result. + +## Deploy + +The deployment spec is a self-contained KServe `LLMInferenceService` template with a +TP4 Decode template, a TP4 Prefill template, InferNex router/scheduler, `InferencePool`, +and the external route. + +| Spec | File | +|---|---| +| P4+D4, KServe + InferNex Bridge | [`gemma4-31b-it-pd-p4d4-llmisvc.yaml`](https://github.com/alauda/aml-docs/tree/master/docs/en/plan/validated_models/assets/gemma4-31b-it/gemma4-31b-it-pd-p4d4-llmisvc.yaml) | + +Before applying it, set the namespace, the two node names, the model PVCs, and registry +references for your cluster. The manifest assumes the P/D pair has equal TP size and that +the two engines can use Ascend Direct transport. + +```bash +base=https://raw.githubusercontent.com/alauda/aml-docs/master/docs/en/plan/validated_models/assets/gemma4-31b-it +kubectl apply -f $base/gemma4-31b-it-pd-p4d4-llmisvc.yaml + +# Production MaaS ingress: +curl -s https:///v1/chat/completions \ + -H "Authorization: Bearer $MAAS_API_KEY" \ + -H 'Content-Type: application/json' \ + -d '{"model":"google/gemma-4-31B-it","messages":[{"role":"user","content":"hello"}]}' +``` + +## Benchmark results + +Closed-loop `aiperf 0.7.0`, driven through MaaS with its API key. Both scenarios run +480 requests per concurrency tier at concurrency 8 / 16 / 32, with warmup equal to the +tier's concurrency. TTFT / E2E are seconds, ITL is ms, Decode is output-only token +throughput, and TPS is input-plus-output token throughput. + +**Scenario ① — fixed-length system-prompt reuse (ISL 8,013 / OSL 128)** + +| Concurrency | TTFT avg (s) | ITL avg (ms) | E2E avg (s) | Decode (tok/s) | Requests/s | TPS (in+out) | +|--:|--:|--:|--:|--:|--:|--:| +| 8 | 4.096 | 29.33 | 7.821 | 129.97 | 1.02 | 8,266.38 | +| 16 | 11.816 | 29.29 | 15.536 | 129.82 | 1.01 | 8,256.63 | +| 32 | 26.843 | 29.36 | 30.572 | 129.84 | 1.01 | 8,257.80 | + +**Scenario ② — multi-turn dialogue (average ISL 17,933 / OSL 128)** + +| Concurrency | TTFT avg (s) | ITL avg (ms) | E2E avg (s) | Decode (tok/s) | Requests/s | TPS (in+out) | +|--:|--:|--:|--:|--:|--:|--:| +| 8 | 14.585 | 28.82 | 18.246 | 55.69 | 0.44 | 7,857.50 | +| 16 | 32.488 | 28.79 | 36.145 | 55.71 | 0.44 | 7,861.46 | +| 32 | 67.295 | 28.85 | 70.959 | 55.72 | 0.44 | 7,862.21 | + +Every tier completed 480 requests with zero errors. Throughput and ITL are flat from +concurrency 8 onward in both scenarios, while TTFT grows with concurrency. The service +is Prefill-capacity-bound at roughly 8k input-dominated tokens/s: increasing average ISL +from 8,013 to 17,933 lowers request throughput from 1.02 to 0.44 requests/s, while total +token throughput changes by about 4.8%. Decode ITL remains stable near 29 ms. + +:::note +The benchmark was sent to MaaS, whose backend selected this P4+D4 proxy. The API key was +injected from a Kubernetes Secret and was not written to the manifest, benchmark log, or +this page. Server metrics and GPU telemetry were disabled because AIPerf 0.7.0 cannot +parse this vLLM metrics histogram; the published values come from completed request +records and CSV export. Every one of the six tiers completed 480 requests with zero errors, +and all four serving Pods remained at zero restarts. + +The delivered scheduler intentionally omits the tokenizer plugin. This topology has only +one fixed Prefill/Decode pair and no prefix-cache producer, so tokenization adds no routing +choice. The tested Hermes Router `26.6.0-rc.3` imposes a 5-second tokenizer initialization +deadline, while this Gemma tokenizer took 17.59 seconds to cold-load. A future multi-pair, +prefix-aware deployment should first use a router build with a configurable initialization +deadline of at least 30 seconds, then restore the tokenizer plugin. +::: + +## Real coding-agent validation + +Gemma-4 was also driven as a real coding agent through the same MaaS gateway, with +reasoning/thinking enabled. **Pi 0.80.6** and **OpenCode 1.17.18** ran the fixed 14-task +`capacity14` Terminal-Bench selection at concurrency **1 / 8 / 32**. Concurrency 1 and 8 +use one attempt per task; concurrency 32 uses three attempts per task, for 42 trials. An +objective task verifier produced the scores below. + +**Pi 0.80.6** + +| Concurrency | Completed trials | Passed | Pass rate | Agent timeouts | Wall time | Correct-task throughput | +|--:|--:|--:|--:|--:|--:|--:| +| 1 | 14 / 14 | 5 / 14 | 35.7% | 0 | 1h 23m 24s | 3.60 tasks/hr | +| 8 | 14 / 14 | 7 / 14 | 50.0% | 1 | 16m 13s | **25.89 tasks/hr** | +| 32 | 42 / 42 | 20 / 42 | 47.6% | 15 | 1h 8m 59s | 17.40 tasks/hr | + +The harness reports **Pass@2 = 54.8%** for Pi concurrency 32. Concurrency 8 is the +best observed Pi operating point: increasing to 32 did not improve pass rate or +correct-task throughput and introduced 15 `AgentTimeoutError` trials. + +**OpenCode 1.17.18** + +| Concurrency | Completed trials | Passed | Pass rate | Agent timeouts | Wall time | Correct-task throughput | +|--:|--:|--:|--:|--:|--:|--:| +| 1 | 14 / 14 | 7 / 14 | 50.0% | 0 | 1h 35m 17s | 4.41 tasks/hr | +| 8 | 14 / 14 | 6 / 14 | 42.9% | 0 | 16m 1s | **22.47 tasks/hr** | +| 32 | 42 / 42 | 15 / 42 | 35.7% | 16 | 50m 35s | 17.79 tasks/hr | + +OpenCode concurrency 32 reports **Pass@2 = 42.9%**. Its best correct-task throughput is +also at concurrency 8, while its highest pass rate is the serial concurrency-1 run. The +concurrency-32 tier adds 16 `AgentTimeoutError` trials and reduces both raw pass rate and +Pass@2, so concurrency 32 is not a suitable operating point for either agent on this +deployment. + +During both Agent sweeps, the Gemma Prefill, Decode, scheduler, and proxy Pods remained +Ready with zero restarts and no serving 5xx/traceback. The timeout trials are therefore +recorded as Agent/task-level failures rather than an engine crash. Pi is stronger at the +throughput-oriented concurrency-8 point (7/14 versus OpenCode 6/14); OpenCode is stronger +at serial concurrency 1 (7/14 versus Pi 5/14). The earlier `mini7` results remain smoke +evidence and are not used as the formal Agent score. diff --git a/docs/en/plan/validated_models/index.mdx b/docs/en/plan/validated_models/index.mdx index 21fe3050..d3abb7f8 100644 --- a/docs/en/plan/validated_models/index.mdx +++ b/docs/en/plan/validated_models/index.mdx @@ -33,6 +33,7 @@ scheduling) see [Deploy](../../deploy/index.mdx). | DeepSeek-V4-Flash (W4A8) | `deepseek_v4` MoE (MLA + DSA + MTP) | 256-expert MoE | W4A8 | Ascend 910B4 ×8 | vLLM-Ascend nightly | [DeepSeek-V4-Flash (W4A8)](./deepseek-v4-flash-w4a8.mdx) | | DeepSeek-V4-Flash (W8A8) | `deepseek_v4` MoE (MLA + DSA + MTP) | 256-expert MoE | W8A8 | Ascend 910B3 ×16 | vLLM-Ascend nightly | [DeepSeek-V4-Flash (W8A8)](./deepseek-v4-flash-w8a8.mdx) | | MiniMax-M2.5 (W8A8) | `minimax_m2` MoE (full-attn, 256-expert) | 230B-A10B | W8A8 (QuaRot) | Ascend 910B3 ×16 | vLLM-Ascend v0.23.0 | [MiniMax-M2.5 (W8A8)](./minimax-m2.5-w8a8.mdx) | +| Gemma-4-31B-it | `gemma4` dense multimodal | 31B | BF16 | Ascend 910B3 ×8 | vLLM-Ascend nightly-main (0721) | [Gemma-4-31B-it](./gemma4-31b-it.mdx) | The two Qwen models and DeepSeek-V4-Flash (W4A8) were validated on Ascend 910B4 (32 GB/card), driven through KServe `LLMInferenceService` with load-aware routing (InferNex-Bridge + @@ -53,6 +54,11 @@ mooncake store and no speculative decoding. Because this model is "fast" (~10B a keeps only the decode graph + local prefix cache. It was additionally validated as a **real coding agent** (OpenCode / Pi) through the product MaaS gateway. +Gemma-4-31B-it is a dense BF16 model validated as a **P4+D4 composite service** across +two 910B3 nodes: TP4 Prefill and TP4 Decode exchange KV through Ascend Direct transport. +It uses eight NPUs in total; its figures are an operating envelope for that P/D service, +not a comparison with a four-card aggregated replica. + ## Runtime images | Engine | Device | Image (validated tag) | Used by | Notes | @@ -61,6 +67,7 @@ keeps only the decode graph + local prefix cache. It was additionally validated | vLLM-Ascend nightly (release-pinned) | Huawei Ascend NPU | `quay.io/ascend/vllm-ascend:nightly-releases-v0.22.1rc-openeuler` | Qwen3.6-27B (W8A8), DeepSeek-V4-Flash (W4A8) | Carries the `qwen3_5` Gated DeltaNet hybrid + MTP **and** the `deepseek_v4` MoE stack — the stock `v0.18.0` cannot load either. Use this **release-pinned** tag, **not** the moving `nightly-main-openeuler` (it drifted to a broken build whose TP workers crash). | | vLLM-Ascend nightly-main (dated snapshot) | Huawei Ascend NPU | `quay.io/ascend/vllm-ascend:nightly-main-openeuler-0709` | DeepSeek-V4-Flash (W8A8) | A **later** `nightly-main` snapshot (vLLM 0.23.0) that carries the DP2 cross-node graph + DSA-CP prefix-cache fixes the W8A8 cross-node deployment needs. Pin a **dated** snapshot like this one, not the moving `nightly-main` tag. | | vLLM-Ascend v0.23.0 (release-pinned) | Huawei Ascend NPU | `quay.io/ascend/vllm-ascend:v0.23.0-openeuler` | MiniMax-M2.5 (W8A8) | The **release-pinned** `nightly-releases-v0.23.0-openeuler` build (vLLM 0.23.0), carrying the `minimax_m2` MoE stack + tool-call/reasoning parsers **and** vLLM #11505 (streaming tool-call arguments held until the closing ``). Prefer this dated **release** tag over the moving `nightly-main`. | +| vLLM-Ascend nightly-main (dated snapshot) | Huawei Ascend NPU | `quay.io/ascend/vllm-ascend:nightly-main-openeuler-0721` | Gemma-4-31B-it | vLLM 0.25.1 / CANN 9.0.1 snapshot used by the validated P4+D4 service. Keep this exact image for the known-good Gemma baseline; it is not a current-version recommendation. | :::tip The Ascend CANN images are arm64. Always match the runtime image's CANN version to @@ -75,6 +82,9 @@ real serving patterns. Output is pinned to **128 tokens** and load is **closed-l concurrency 4** (4 in-flight requests, fixed); each scenario ran 240 requests. DeepSeek-V4-Flash (W8A8) and MiniMax-M2.5 (W8A8) are the exceptions — they were swept at **concurrency 8 / 16 / 32** (480 requests per tier). +Gemma-4-31B-it uses both scenarios and the same concurrency sweep, but is sent directly +to its P4+D4 proxy because the MaaS backend was still attached to the older aggregate +service at measurement time. | Scenario | What it models | Dataset | Request shape | |---|---|---|---|