diff --git a/.gitignore b/.gitignore index 0234304a1..07532fa9a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ yarn-error.log* # wrangler local dev .dev.vars .wrangler/ + +# Scratch notes for unpublished posts, never rendered by lib/blog.js +/content/blog/_drafts/ diff --git a/components/MoeExpertRoutingAnimation.jsx b/components/MoeExpertRoutingAnimation.jsx new file mode 100644 index 000000000..2c6424166 --- /dev/null +++ b/components/MoeExpertRoutingAnimation.jsx @@ -0,0 +1,290 @@ +const gpus = [0, 1, 2, 3]; +const PER_GPU = 32; + +export default function MoeExpertRoutingAnimation() { + return ( +
+ +
+

Expert routing animation

+

+ Why a 235B model only does 22B of work per token +

+

+ Every layer of this model has 128 small expert networks, and a tiny router picks just 8 of + them for each token. The other 120 sit still. That is the whole trick of a mixture of + experts: you pay for 235B parameters in memory, but only about 22B of arithmetic per token. +

+ +
+
+ one token arrives + it has already been through attention for this layer +
+ +
router scores all 128 experts, keeps the top 8
+ +
+ {gpus.map((gpu) => { + // 2 of this GPU's 32 experts are picked, so 8 across 4 GPUs + const hot = [3 + gpu, 18 + ((gpu * 5) % 10)]; + return ( +
+

+ GPU {gpu} + experts {gpu * PER_GPU}-{gpu * PER_GPU + PER_GPU - 1} +

+ + ); + })} +
+ +
+
+ In memory + 235B params + + All 128 experts per layer must be resident, which is why the model is big + +
+
+ Active per token + 22B params + + Only the 8 chosen experts do arithmetic, so it runs like a much smaller model + +
+
+ What this costs you + a network hop + + With expert parallelism the token travels to whichever GPU owns its expert, then the + answer travels back + +
+
+
+
+
+ Counts are from the model config: 128 experts per layer, 8 per token, 94 layers. Splitting 128 + experts over 4 GPUs gives 32 each, so on average 2 experts per GPU fire for any given token. + That average is the catch, because routing is not guaranteed to be even. +
+
+ ); +} diff --git a/components/MultiGpuMemoryFitAnimation.jsx b/components/MultiGpuMemoryFitAnimation.jsx new file mode 100644 index 000000000..588ca697e --- /dev/null +++ b/components/MultiGpuMemoryFitAnimation.jsx @@ -0,0 +1,390 @@ +const cases = [ + { + key: 'one', + verdict: 'fail', + title: '1 GPU', + flag: 'will not start', + note: '221 GiB of weights against an 85.51 GiB budget', + parts: [{ label: 'weights', value: '221 GiB', width: '92.08%', color: '#ef4444' }], + log: 'the model is 2.6x larger than the whole budget\nthere is no flag that fixes this', + }, + { + key: 'two', + verdict: 'fail', + title: '2 GPUs', + flag: 'CUDA out of memory', + note: 'about 110 GiB per card, still too much', + parts: [{ label: 'weights per card', value: '110 GiB', width: '46.04%', color: '#f59e0b' }], + log: 'Failed to load model - not enough GPU memory\n95.01 GiB total, of which 438.31 MiB is free', + }, + { + key: 'four', + verdict: 'pass', + title: '4 GPUs', + flag: '621,392 tokens', + note: 'weights fit, with room for about 19 concurrent 32k conversations', + parts: [ + { label: 'weights', value: '55.19 GiB', width: '23.00%', color: '#0098cc' }, + { label: 'KV cache', value: '27.85 GiB', width: '11.60%', color: '#2bb534' }, + ], + log: 'Worker_TP0 Model loading took 55.19 GiB\nCurrent kv cache memory in use is 27.85 GiB\nGPU KV cache size: 621,392 tokens', + }, +]; + +export default function MultiGpuMemoryFitAnimation() { + return ( +
+ +
+

Memory fit animation

+

+ The same model on 1, 2 and 4 GPUs +

+

+ Every number here came out of a real run. All three bars are drawn to the same scale, and + the dashed line is the 85.51 GiB that vLLM may use on one card at + --gpu-memory-utilization 0.90. A bar reaching past that line means the model does not fit. + Watch it shrink as GPUs are added, and note that it takes 4 before the bar finally lands to + the left of the line. +

+ +
+ {cases.map((c) => ( +
+
+ + {c.title} + {c.note} + + {c.flag} +
+ +
+ what one card must hold + full axis = 240 GiB +
+ +
+ +
+ {c.parts.map((p, i) => ( +
+ + {p.label} {p.value} + +
+ ))} +
+
+ +
{c.log}
+
+ ))} + +
+
+ KV per token, whole model + 188 KiB + 2 x 94 layers x 4 kv heads x 128 head_dim x 2 bytes +
+
+ Per card at TP=4 + 47 KiB + + each card keeps 1 of the 4 kv heads, so the cache divides rather than repeats + +
+
+ Predicted vs reported + 621,337 / 621,392 + + 27.85 GiB divided by 47 KiB, against what vLLM actually printed + +
+
+
+
+
+ Measured on 4x RTX PRO 6000 Blackwell with Qwen3-235B-A22B-Instruct-2507-FP8 on vLLM 0.27.1. + The 1 GPU and 2 GPU bars are what the run actually attempted before failing, not estimates. + Because this model has only 4 key/value heads, its cache is unusually cheap, which is why 4 + cards leave room for about 19 concurrent conversations at the 32,768-token limit we set. +
+
+ ); +} diff --git a/components/MultiGpuSplitModesAnimation.jsx b/components/MultiGpuSplitModesAnimation.jsx new file mode 100644 index 000000000..e58e6c224 --- /dev/null +++ b/components/MultiGpuSplitModesAnimation.jsx @@ -0,0 +1,279 @@ +const modes = [ + { + key: 'tp', + name: 'Tensor parallelism', + flag: '--tensor-parallel-size', + plain: 'Cut every layer into vertical strips. Each GPU holds a strip of all 94 layers.', + talks: 'A lot. Twice per layer, so 188 times per token.', + good: 'Fastest for a single user, because all 4 GPUs work on the same token.', + color: '#0098cc', + }, + { + key: 'pp', + name: 'Pipeline parallelism', + flag: '--pipeline-parallel-size', + plain: 'Cut the stack into horizontal blocks. With 94 layers over 4 GPUs, each one owns about 23 of them.', + talks: 'Barely. One handoff between neighbours per token.', + good: 'Kind to a slow network between GPUs, but a GPU waits its turn.', + color: '#2bb534', + }, + { + key: 'ep', + name: 'Expert parallelism', + flag: '--enable-expert-parallel', + plain: 'Deal the 128 experts out like cards. Each GPU keeps 32 of them, whole.', + talks: 'Medium. Tokens are shipped to whichever GPU owns the expert they need.', + good: 'Only exists for MoE models, and it is how the really big ones are served.', + color: '#a855f7', + }, +]; + +export default function MultiGpuSplitModesAnimation() { + return ( +
+ +
+

Three ways to split animation

+

+ The same model, cut three different ways across four GPUs +

+

+ These are not competing products, they are three different cuts through the same pile of + weights, and you can combine them. Each box below is one GPU. Watch which parts light up, + because that tells you which GPUs are doing work at the same moment. +

+ +
+ {modes.map((mode) => ( +
+

{mode.name}

+ {mode.flag} + + + +
+

+ What it does + {mode.plain} +

+

+ How much it talks + {mode.talks} +

+

+ When it wins + {mode.good} +

+
+
+ ))} +
+
+
+ Layer and expert counts are Qwen3-235B-A22B: 94 layers, 128 experts with 8 picked per token. + Under tensor parallelism all four GPUs light up together on every token. Under pipeline + parallelism they light up in turn, which is the idle time you are trading away. +
+
+ ); +} diff --git a/components/MultiGpuTensorSplitAnimation.jsx b/components/MultiGpuTensorSplitAnimation.jsx new file mode 100644 index 000000000..2e9bbe573 --- /dev/null +++ b/components/MultiGpuTensorSplitAnimation.jsx @@ -0,0 +1,374 @@ +const steps = [ + { label: 'A token arrives', detail: 'all 4 GPUs get the same copy of it' }, + { label: 'Split sideways', detail: 'each GPU owns 16 of the 64 attention heads' }, + { label: 'Work alone', detail: 'no GPU needs to ask the others anything yet' }, + { label: 'Partial answers', detail: 'each GPU has a quarter of the answer' }, + { label: 'Add them up', detail: 'one all-reduce, and all 4 hold the full result' }, +]; + +export default function MultiGpuTensorSplitAnimation() { + return ( +
+ +
+

Tensor parallelism animation

+

+ One layer, sliced four ways +

+

+ This is the part people usually get wrong, so it is worth being precise. The weights get + divided, and the thing flowing through them does not. Every GPU starts each layer holding + an identical copy of the token, does a quarter of the arithmetic on its own slice of the + weights, and ends up with a quarter of an answer. Then they add their quarters together. +

+ +
+
+ the token, 4096 numbers wide + copied to all four GPUs, not divided +
+ +
+ {[0, 1, 2, 3].map((gpu) => ( +
+

+ GPU {gpu} + heads {gpu * 16}-{gpu * 16 + 15} +

+ + ))} +
+ +
+ + +
+ the finished layer output, now identical on all four GPUs + and the next layer does the whole dance again +
+
+ +
+ {steps.map((step, i) => ( +
+ Step {i + 1} + {step.label} + {step.detail} +
+ ))} +
+
+
+ Shapes are Qwen3-235B-A22B: hidden size 4096, 64 attention heads, 4 key/value heads, 94 + layers. Those 4 key/value heads are the reason this model cannot be split cleanly more than 4 + ways, which we come back to later. +
+
+ ); +} diff --git a/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md b/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md new file mode 100644 index 000000000..2fb3809ea --- /dev/null +++ b/content/blog/running-a-big-llm-across-multiple-gpus-with-vllm.md @@ -0,0 +1,632 @@ +--- +title: "Running a big LLM across multiple GPUs with vLLM" +seoTitle: "Running a big LLM across multiple GPUs with vLLM" +seoDescription: "A runbook for serving a model too big for one GPU: download to serving in seven steps, with every vLLM flag, startup log line and real error explained, plus tensor, pipeline and expert parallelism benchmarked head to head on a 235B model across four RTX PRO 6000 cards." +datePublished: 2026-09-01T10:00:00.000Z +slug: running-a-big-llm-across-multiple-gpus-with-vllm +author: shubham-katara +authors: ["shubham-katara", "saiyam-pathak"] +cover: /img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png +tags: ["vllm", "gpu", "nvidia", "llm", "platform-engineering"] +sponsor: + name: Utho + url: "https://utho.com/?utm_source=Kubesimplify&utm_medium=docs&utm_campaign=Saiyam" + # logoLight = navy mark (shown on light theme); logoDark = white mark (shown on dark theme) + logoLight: /img/sponsors/utho-logo-light.png + logoDark: /img/sponsors/utho-logo-dark.png + blurb: "Every number in this runbook was measured on an 8x NVIDIA RTX PRO 6000 Blackwell node from Utho Cloud. If you need GPU infrastructure to run workloads like these, take a look." +--- + +Sooner or later everyone running models locally hits the same wall. You find a model you want, you look at the download size, and it is bigger than the GPU you own. A 235B model needs roughly 236 GB just for its weights. The card we have holds 96 GB. A handful of current data-centre parts do carry more, but nothing on our machine does, and no amount of clever flags will make 236 GB squeeze into 96 GB. + +The answer is to use more than one GPU. That part everybody knows. The part that is genuinely confusing is what "use more than one GPU" actually means. Does each GPU get a copy of the model? Does the model get cut in half? Do the GPUs take turns? Which of those is happening, and what does it cost you? + +Let's answer that properly, with a real model on real hardware. + +## What this post covers + +This is the runbook. Seven steps, from downloading a 236 GB model to serving it across four GPUs, with every command, flag, startup log line and real error explained. It is written for the person with root on the box, and it assumes no prior knowledge of distributed computing: if you know what a GPU is and you have run a model locally once, you are qualified. + +The theory arrives where you need it to make a decision, not before. Step 3 explains what a tensor-parallel split actually costs, because that is where you pick one, and Step 6 explains why the three options trade against each other, because that is where you read the numbers. Nothing here is theory for its own sake. + +## The machine and the model + +Numbers mean nothing without the hardware attached, so here it is once. + +**The machine:** a server with 8x NVIDIA RTX PRO 6000 Blackwell Server Edition cards. Each card has 96 GB of memory, and the machine reports 95.01 GiB of that as usable. We borrowed 4 of the 8 cards for this work. + +One detail that matters more than it looks: these GPUs are **not** connected by NVLink. NVLink is NVIDIA's fast direct GPU-to-GPU cable. Without it, GPUs talk to each other over PCIe and through the CPU, which is slower. You can check what you have with one command: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# nvidia-smi topo -m +``` + +| Device | GPU0 | GPU1 | GPU2 | GPU3 | GPU4 | GPU5 | GPU6 | GPU7 | NIC0 | CPU Affinity | NUMA Affinity | +| :------- | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :-------------- | :-----------: | +| **GPU0** | X | SYS | SYS | SYS | SYS | SYS | SYS | SYS | SYS | 48-55,176-183 | 6 | +| **GPU1** | SYS | X | SYS | SYS | SYS | SYS | SYS | SYS | PHB | 32-39,160-167 | 4 | +| **GPU2** | SYS | SYS | X | SYS | SYS | SYS | SYS | SYS | SYS | 0-7,128-135 | 0 | +| **GPU3** | SYS | SYS | SYS | X | SYS | SYS | SYS | SYS | SYS | 16-23,144-151 | 2 | +| **GPU4** | SYS | SYS | SYS | SYS | X | SYS | SYS | SYS | SYS | 112-119,240-247 | 14 | +| **GPU5** | SYS | SYS | SYS | SYS | SYS | X | SYS | SYS | SYS | 96-103,224-231 | 12 | +| **GPU6** | SYS | SYS | SYS | SYS | SYS | SYS | X | SYS | SYS | 64-71,192-199 | 8 | +| **GPU7** | SYS | SYS | SYS | SYS | SYS | SYS | SYS | X | SYS | 80-87,208-215 | 10 | +| **NIC0** | SYS | PHB | SYS | SYS | SYS | SYS | SYS | SYS | X | | | + +The legend that command prints, trimmed to the codes that matter here: + +| Symbol | Meaning | +| :----- | :---------------------------------------------------------------------------- | +| `X` | Self | +| `SYS` | Across PCIe **and** the interconnect between CPU sockets. The slowest option. | +| `NODE` | Across PCIe and the bridges inside one NUMA node | +| `PHB` | Across PCIe and a PCIe host bridge, typically the CPU | +| `PXB` | Across multiple PCIe bridges, without touching the host bridge | +| `PIX` | Across at most a single PCIe bridge. The fastest non-NVLink option. | +| `NV#` | Across a bonded set of `#` NVLinks | + +On our machine every pair of GPUs reports `SYS`, which means the traffic goes across PCIe and then across the link between the CPU sockets. If you had NVLink you would see `NV1`, `NV2` and so on instead. Keep this in mind, because it changes which splitting method is fastest. + +**The model:** `Qwen/Qwen3-235B-A22B-Instruct-2507-FP8`. Let's unpack that name, because it is doing a lot of work: + +- **235B** is the total parameter count, 235 billion. +- **A22B** means 22 billion **active** parameters. This is a mixture-of-experts model: each layer holds 128 small expert networks and a router picks just 8 of them per token, so you pay for 235B in memory but only about 22B in arithmetic. +- **FP8** is the number format the weights are stored in, 8 bits each, so one byte per parameter. + +**The software:** vLLM 0.27.1 running in the official container, with PyTorch 2.13.0 and CUDA 13.0, on driver 610.43.02. + +--- + +## Step 1: Getting the model onto the machine + +Before anything can be split across GPUs it has to be on disk, and with a model this size that is not a formality. + +### Check your disk first + +A quarter of a terabyte has to land somewhere. Run `df -h` before you start, and if the machine is shared, leave real headroom rather than just enough: platforms that manage disk as a resource start taking action well before the disk is actually full. + +### The download + +With the headroom confirmed, you download it with the Hugging Face CLI: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# pip install huggingface_hub hf_transfer +root@utho-gpu-rtxpro6000-8-62383:~# HF_XET_HIGH_PERFORMANCE=1 hf download Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 +Downloading bytes: ████████████████████████████████████████████████▏ | 24.4GB, 234MB/s +Reconstructing (incomplete total...): 13%|███████████████▋ | 10.0GB / 80.0GB, 104MB/s +Fetching 34 files: 0%| | 0/34 [00:00 https://blog.kubesimplify.com/ - 2026-08-18T08:33:37.156Z + 2026-09-01T11:10:35.934Z Kubesimplify hello@kubesimplify.com + + Running a big LLM across multiple GPUs with vLLM + + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + 2026-09-01T10:00:00.000Z + 2026-09-01T10:00:00.000Z + A runbook for serving a model too big for one GPU: download to serving in seven steps, with every vLLM flag, startup log line and real error explained, plus tensor, pipeline and expert parallelism benchmarked head to head on a 235B model across four RTX PRO 6000 cards. + + + + + + + + Zero Trust in Practice: Migrating from Istio Sidecar to Ambient Mode + + https://blog.kubesimplify.com/zero-trust-istio-sidecar-vs-ambient + 2026-08-31T10:00:00.000Z + 2026-08-31T10:00:00.000Z + A hands-on comparison of Istio sidecar and ambient mode for zero-trust service mesh. Same app, same policy, two architectures proven step by step on a local cluster. + + + + + + + + Running Qwen3.8-Flash-Next on a DGX Spark and RTX PRO 6000 + + https://blog.kubesimplify.com/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000 + 2026-08-27T06:30:00.000Z + 2026-08-27T06:30:00.000Z + + + + + + + The Local LLM Glossary: Every Term, Flag, and Number in Plain English diff --git a/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cake-layers.png b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cake-layers.png new file mode 100644 index 000000000..8e7f7d3af Binary files /dev/null and b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cake-layers.png differ diff --git a/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png new file mode 100644 index 000000000..6ed8b13f6 Binary files /dev/null and b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.png differ diff --git a/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg new file mode 100644 index 000000000..9c6a3d465 --- /dev/null +++ b/public/img/blog/running-a-big-llm-across-multiple-gpus-with-vllm/cover.svg @@ -0,0 +1,55 @@ + + + +One big model, four GPUs +how a 235B model is cut up so it fits, and what that costs + +ONE CARD + + + +95 GiB +usable + + + +236 GB of weights +2.3x too big +no flag fixes this + +FOUR CARDS, --tensor-parallel-size 4 + + + +GPU 0 +59 GB +weights +16 of 64 heads + + + +GPU 1 +59 GB +weights +16 of 64 heads + + + +GPU 2 +59 GB +weights +16 of 64 heads + + + +GPU 3 +59 GB +weights +16 of 64 heads + +188 all-reduces per token + +QWEN3-235B-A22B FP8 - 128 EXPERTS, 8 PER TOKEN - vLLM 0.27.1 +tensor, pipeline and expert parallelism explained in plain english +blog.kubesimplify.com + \ No newline at end of file diff --git a/public/llms-full.txt b/public/llms-full.txt index df6842053..bf7a9a900 100644 --- a/public/llms-full.txt +++ b/public/llms-full.txt @@ -5,6 +5,1654 @@ --- +# Running a big LLM across multiple GPUs with vLLM + +- Canonical: https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm +- Published: 2026-09-01 +- Summary: A runbook for serving a model too big for one GPU: download to serving in seven steps, with every vLLM flag, startup log line and real error explained, plus tensor, pipeline and expert parallelism benchmarked head to head on a 235B model across four RTX PRO 6000 cards. + +Sooner or later everyone running models locally hits the same wall. You find a model you want, you look at the download size, and it is bigger than the GPU you own. A 235B model needs roughly 236 GB just for its weights. The card we have holds 96 GB. A handful of current data-centre parts do carry more, but nothing on our machine does, and no amount of clever flags will make 236 GB squeeze into 96 GB. + +The answer is to use more than one GPU. That part everybody knows. The part that is genuinely confusing is what "use more than one GPU" actually means. Does each GPU get a copy of the model? Does the model get cut in half? Do the GPUs take turns? Which of those is happening, and what does it cost you? + +Let's answer that properly, with a real model on real hardware. + +## What this post covers + +This is the runbook. Seven steps, from downloading a 236 GB model to serving it across four GPUs, with every command, flag, startup log line and real error explained. It is written for the person with root on the box, and it assumes no prior knowledge of distributed computing: if you know what a GPU is and you have run a model locally once, you are qualified. + +The theory arrives where you need it to make a decision, not before. Step 3 explains what a tensor-parallel split actually costs, because that is where you pick one, and Step 6 explains why the three options trade against each other, because that is where you read the numbers. Nothing here is theory for its own sake. + +## The machine and the model + +Numbers mean nothing without the hardware attached, so here it is once. + +**The machine:** a server with 8x NVIDIA RTX PRO 6000 Blackwell Server Edition cards. Each card has 96 GB of memory, and the machine reports 95.01 GiB of that as usable. We borrowed 4 of the 8 cards for this work. + +One detail that matters more than it looks: these GPUs are **not** connected by NVLink. NVLink is NVIDIA's fast direct GPU-to-GPU cable. Without it, GPUs talk to each other over PCIe and through the CPU, which is slower. You can check what you have with one command: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# nvidia-smi topo -m +``` + +| Device | GPU0 | GPU1 | GPU2 | GPU3 | GPU4 | GPU5 | GPU6 | GPU7 | NIC0 | CPU Affinity | NUMA Affinity | +| :------- | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :--: | :-------------- | :-----------: | +| **GPU0** | X | SYS | SYS | SYS | SYS | SYS | SYS | SYS | SYS | 48-55,176-183 | 6 | +| **GPU1** | SYS | X | SYS | SYS | SYS | SYS | SYS | SYS | PHB | 32-39,160-167 | 4 | +| **GPU2** | SYS | SYS | X | SYS | SYS | SYS | SYS | SYS | SYS | 0-7,128-135 | 0 | +| **GPU3** | SYS | SYS | SYS | X | SYS | SYS | SYS | SYS | SYS | 16-23,144-151 | 2 | +| **GPU4** | SYS | SYS | SYS | SYS | X | SYS | SYS | SYS | SYS | 112-119,240-247 | 14 | +| **GPU5** | SYS | SYS | SYS | SYS | SYS | X | SYS | SYS | SYS | 96-103,224-231 | 12 | +| **GPU6** | SYS | SYS | SYS | SYS | SYS | SYS | X | SYS | SYS | 64-71,192-199 | 8 | +| **GPU7** | SYS | SYS | SYS | SYS | SYS | SYS | SYS | X | SYS | 80-87,208-215 | 10 | +| **NIC0** | SYS | PHB | SYS | SYS | SYS | SYS | SYS | SYS | X | | | + +The legend that command prints, trimmed to the codes that matter here: + +| Symbol | Meaning | +| :----- | :---------------------------------------------------------------------------- | +| `X` | Self | +| `SYS` | Across PCIe **and** the interconnect between CPU sockets. The slowest option. | +| `NODE` | Across PCIe and the bridges inside one NUMA node | +| `PHB` | Across PCIe and a PCIe host bridge, typically the CPU | +| `PXB` | Across multiple PCIe bridges, without touching the host bridge | +| `PIX` | Across at most a single PCIe bridge. The fastest non-NVLink option. | +| `NV#` | Across a bonded set of `#` NVLinks | + +On our machine every pair of GPUs reports `SYS`, which means the traffic goes across PCIe and then across the link between the CPU sockets. If you had NVLink you would see `NV1`, `NV2` and so on instead. Keep this in mind, because it changes which splitting method is fastest. + +**The model:** `Qwen/Qwen3-235B-A22B-Instruct-2507-FP8`. Let's unpack that name, because it is doing a lot of work: + +- **235B** is the total parameter count, 235 billion. +- **A22B** means 22 billion **active** parameters. This is a mixture-of-experts model: each layer holds 128 small expert networks and a router picks just 8 of them per token, so you pay for 235B in memory but only about 22B in arithmetic. +- **FP8** is the number format the weights are stored in, 8 bits each, so one byte per parameter. + +**The software:** vLLM 0.27.1 running in the official container, with PyTorch 2.13.0 and CUDA 13.0, on driver 610.43.02. + +--- + +## Step 1: Getting the model onto the machine + +Before anything can be split across GPUs it has to be on disk, and with a model this size that is not a formality. + +### Check your disk first + +A quarter of a terabyte has to land somewhere. Run `df -h` before you start, and if the machine is shared, leave real headroom rather than just enough: platforms that manage disk as a resource start taking action well before the disk is actually full. + +### The download + +With the headroom confirmed, you download it with the Hugging Face CLI: + +```bash +root@utho-gpu-rtxpro6000-8-62383:~# pip install huggingface_hub hf_transfer +root@utho-gpu-rtxpro6000-8-62383:~# HF_XET_HIGH_PERFORMANCE=1 hf download Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 +Downloading bytes: ████████████████████████████████████████████████▏ | 24.4GB, 234MB/s +Reconstructing (incomplete total...): 13%|███████████████▋ | 10.0GB / 80.0GB, 104MB/s +Fetching 34 files: 0%| | 0/34 [00:00 **Note on the demo architecture:** For this demonstration each "service" is deployed as `kennethreitz/httpbin`: an echo server that reflects back request headers. This lets us inspect mTLS identity headers directly. The actual policy tests are performed by running `curl` from temporary pods or by `kubectl exec` into the `frontend` pod which uses `curlimages/curl`. The app logic is irrelevant. What matters is whether the mesh allows or blocks the traffic. + +--- + +## Two Architectures, One Goal + +**Sidecar mode** has been Istio's model since 2017. Every pod that joins the mesh gets a second container injected into it, an Envoy proxy running as `istio-proxy`. A one-time init container installs iptables rules inside the pod's own network namespace so every byte in or out of your app container gets silently rerouted through that sidecar first. The sidecar terminates and originates mTLS, holds that pod's certificate and enforces whatever `AuthorizationPolicy` applies to it. Your application code never changes. But every pod whether or not it ever handles a sensitive request now carries a full proxy. + +**Ambient mode** splits that same job into two layers instead of bolting a proxy onto every pod. A `ztunnel` runs once per node not once per pod as a DaemonSet. It handles mTLS and workload identity for every pod scheduled on that node using an HTTP CONNECT-based tunnel protocol called HBONE to talk to other nodes. It does not read HTTP. It has no concept of a path or a method. For that ambient adds a second optional component: a **waypoint**, the exact same Envoy binary the sidecar uses but deployed as its own independent workload attached only to the specific service that actually needs L7 rules. + +In practice this changes how you join the mesh, how you write policy and what you're troubleshooting when the system doesn't do what it says. The rest of this post is that difference proven step by step on the same app on the same cluster. + +--- + +## The Project + +To make the comparison honest I set one constraint: the same application, the same intended policy under both architectures so nothing could be explained away by "the app was different." + +The app is deliberately small and one design detail that matters: **each service gets its own Kubernetes ServiceAccount** not a shared one. Istio's identity model is built entirely on the ServiceAccount a pod runs as not the pod itself. If all four services shared one ServiceAccount there'd be no way to write a policy that says "only orders may call payments" because Istio would have no way to tell orders traffic apart from frontend's. Four ServiceAccounts is what makes the whole zero-trust story expressible at all. + +The target policy is narrow on purpose: **payments only accepts POST requests to `/post` and only from orders.** Everything else including a direct call from frontend gets denied. That one rule gets implemented twice: once as a sidecar-mode policy and once as an ambient-mode one on the same cluster torn down cleanly between runs so neither phase could quietly lean on leftovers from the other. + +--- + +## Standing Up the Cluster + +A local kind cluster is enough for this: + +```bash +kind create cluster --name zt-demo +``` + +```bash +kubectl get nodes +``` + +One node, one control plane, `Ready` status. No Istio components exist yet. + +--- + +## Deploying the Baseline: No Mesh at All + +```bash +kubectl apply -f app/ +``` + +![Pods starting up with no mesh](/img/blog/zero-trust-istio-sidecar-vs-ambient/01-pods-no-mesh.png) + +Four services coming up with zero Istio anywhere in the cluster. `READY 1/1`: one container, no sidecar because there's no mesh to inject one yet. + +At this point calling payments directly from frontend with no policy anywhere just worked. No mesh means no gate. That's the baseline everything else in this post is measured against. + +```bash +kubectl -n zt-demo exec deploy/frontend -- curl -s http://payments/post -X POST -d '{"amount": 500}' +``` + +```json +{ + "args": {}, + "data": "", + "files": {}, + "form": { + "{\"amount\": 500}": "" + }, + "headers": { + "Accept": "*/*", + "Content-Length": "15", + "Content-Type": "application/x-www-form-urlencoded", + "Host": "payments", + "User-Agent": "curl/8.21.0" + }, + "json": null, + "origin": "10.244.0.8", + "url": "http://payments/post" +} +``` + +The response is plain HTTP. No `X-Forwarded-Client-Cert` header. No encryption. No identity. The `origin` field shows the raw pod IP (`10.244.0.8`). + +--- + +## Phase 1: Sidecar Mode, Step by Step + +### Install and Inject + +```bash +istioctl install --set profile=minimal -y +kubectl label namespace zt-demo istio-injection=enabled --overwrite +kubectl -n zt-demo rollout restart deployment orders payments inventory frontend +``` + +```text +✓ Istio core installed +✓ Istiod installed +✓ Installation complete +namespace/zt-demo labeled +deployment.apps/orders restarted +deployment.apps/payments restarted +deployment.apps/inventory restarted +deployment.apps/frontend restarted +``` + +Labeling a namespace for sidecar injection does nothing to pods that already exist. Kubernetes has no mechanism to add a container to a running pod so every workload has to be recreated. Watch the `READY` column: pods that were `1/1` a moment ago come back `2/2`. The `0/2` pending rows are pods still finishing sidecar startup. This is the first real operational cost of sidecar mode and it's visible directly in the pod list. + +```bash +kubectl -n zt-demo wait --for=condition=Ready pod -l app=inventory --timeout=600s +kubectl -n zt-demo wait --for=condition=Ready pod -l app=orders --timeout=600s +kubectl -n zt-demo wait --for=condition=Ready pod -l app=payments --timeout=600s +kubectl -n zt-demo get pods +``` + +```text +NAME READY STATUS RESTARTS AGE +frontend-599cd6b667-8sw7c 2/2 Running 0 40s +inventory-6656996d9d-k798x 2/2 Running 0 40s +orders-858bc67b6-txzzq 2/2 Running 0 40s +payments-5cbcb64d66-6jmws 2/2 Running 0 40s +``` + +All pods now at `2/2`. Every pod carries its own proxy. + +### mTLS Is Already Working, Before Any Policy Says So + +```bash +kubectl apply -f - < **Transparent proxy note:** application code always sends plain `http://` to its local proxy. The sidecar transparently upgrades the connection to mutual TLS across the wire, you never change application code to `https://`. + +> **What's `127.0.0.6`?** It's Envoy's internal loopback redirect IP, used in sidecar mode only. The iptables rules installed inside the pod redirect all outbound traffic through the local Envoy proxy first, so the upstream application sees `127.0.0.6` as the source instead of the real pod IP. Hold onto that, it flips to the real pod IP once we get to ambient mode, where there's no per-pod proxy to redirect through. + +### Locking It Down: The Sidecar AuthorizationPolicy + +Before applying one, here's the anatomy of an Istio `AuthorizationPolicy`: + +- **`action`** - `ALLOW` or `DENY` +- **`selector` / `targetRefs`** - which workload or Gateway this policy attaches to +- **`rules`** + - **`from`** - source identities (`principals`) allowed to connect + - **`to`** - operations allowed: HTTP methods, paths, or ports + - **`when`** - optional extra conditions +The policy below attaches directly to the workload using `selector.matchLabels`. The sidecar inside the `payments` pod evaluates this rule. The `principals` field references the SPIFFE identity derived from the `orders` ServiceAccount. + +```bash +kubectl apply -f - < **Note on ambient mTLS defaults:** In ambient mode ztunnel automatically encrypts in-mesh traffic using mTLS. You only need a `PeerAuthentication` resource if you want to explicitly control the mode (for example `PERMISSIVE` to allow plaintext from outside the mesh or `STRICT` to reject anything non-mTLS). For this demo we rely on the ambient default. + +### mTLS Active by Default + +```bash +kubectl -n zt-demo exec deploy/frontend -- curl -s http://payments/post -X POST -d '{"amount": 500}' +``` + +```json +{ + "args": {}, + "data": "", + "files": {}, + "form": { + "{\"amount\": 500}": "" + }, + "headers": { + "Accept": "*/*", + "Content-Length": "15", + "Content-Type": "application/x-www-form-urlencoded", + "Host": "payments", + "User-Agent": "curl/8.21.0" + }, + "json": null, + "origin": "10.244.0.20", + "url": "http://payments/post" +} +``` + +The call succeeds (`200`) confirming ztunnel is encrypting traffic. But notice: no `X-Forwarded-Client-Cert` header and the `origin` is the real pod IP (`10.244.0.20`), not `127.0.0.6`. In sidecar mode the destination proxy injects the identity header and redirects through localhost. In ambient mode without a waypoint, ztunnel handles encryption at L4 without touching HTTP headers. The identity is still cryptographically verified - you just can't see it in the HTTP response yet. + +### Where Ambient Draws Its Line: The Fail-Safe Behavior + +Here is the critical learning moment. When applying the exact same `AuthorizationPolicy` shape that worked cleanly in sidecar mode directly in ambient mode, it got accepted but with a warning attached to its status field: + +```bash +kubectl apply -f - < **`000` vs `403`:** `000` is what curl prints for `%{http_code}` when it never receives an HTTP response at all - here, because ztunnel dropped the TCP connection at Layer 4 before any HTTP exchange could happen. `403` is an actual HTTP response returned by a Layer 7 proxy (like Envoy) after it inspected the request and rejected it. In the frontend test below, curl's own process **exit code** is `56` ("failure in receiving network data") - a separate number from the `000` status placeholder, and further confirmation that the connection was cut, not answered. + +```bash +# Test from orders +kubectl run curl-orders -n zt-demo --image=curlimages/curl --restart=Never \ + --overrides='{"spec":{"serviceAccountName":"orders"}}' \ + -- curl -s -o /dev/null -w '%{http_code}\n' \ + http://payments.zt-demo.svc.cluster.local/post -X POST -d '{"amount": 500}' +``` + +```text +000 +``` + +```bash +# Test from frontend +kubectl -n zt-demo exec deploy/frontend -- \ + curl -s -o /dev/null -w '%{http_code}\n' --max-time 5 \ + http://payments/post -X POST -d '{"amount": 500}' +``` + +```text +000 +command terminated with exit code 56 +``` + +Both denied. Orders with the correct identity and frontend without it both get blocked. ztunnel is L4-only by design. L4 gives you identity-based rules like "A can call B" which is exactly what ztunnel enforces. The path-and-method rule I wrote needed the L7 layer which is exactly what the waypoint below exists to provide. + +![Istio L4 vs L7 security comparison](/img/blog/zero-trust-istio-sidecar-vs-ambient/l4-l7-security-table.png) + +### The Bridge: Why Waypoints Use Gateway API + +ztunnel handles Layer 4 (TCP + mTLS) only. It secures the wire and authenticates peers, but it cannot look inside HTTP requests. To enforce policies based on HTTP paths, methods, or headers, Ambient Mesh deploys an on-demand Envoy pod called a **Waypoint**. Istio models waypoints using the standard Kubernetes Gateway API resources rather than inventing a new CRD. The Waypoint acts as an L7 proxy for a specific service, sitting in the data path only when needed. + +### Bringing in a Waypoint for the One Service That Needs It + +```bash +# Install Gateway API CRDs first +kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.1.0/standard-install.yaml + +# Create the waypoint +istioctl waypoint apply --namespace zt-demo --name payments-waypoint --for service +``` + +```text +customresourcedefinition.apiextensions.k8s.io/gatewayclasses.gateway.networking.k8s.io created +customresourcedefinition.apiextensions.k8s.io/gateways.gateway.networking.k8s.io created +... +✓ waypoint zt-demo/payments-waypoint applied +``` + +Only payments gets a waypoint. Frontend, orders and inventory never do because none of them need HTTP-level policy. ztunnel's L4 identity and encryption is all they ever require. + +A waypoint is not a custom Istio object. It is a standard Kubernetes Gateway API resource. Here is what gets created: + +```yaml +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: payments-waypoint + namespace: zt-demo +spec: + gatewayClassName: istio-waypoint + listeners: + - name: mesh + port: 15008 + protocol: HBONE +``` + +Waypoints plug into the same Gateway API model Kubernetes already has rather than inventing a new one. + +Once istiod finishes reconciling it, `istioctl waypoint status` confirms it: `Programmed`, assigned to `payments-waypoint.zt-demo.svc.cluster.local:15008`, ready to receive traffic. + +### The Full Picture, Running + +```bash +echo '--- namespace labels ---' && kubectl get namespace zt-demo --show-labels +echo '--- app pods ---' && kubectl -n zt-demo get pods -o wide +echo '--- ztunnel pods ---' && kubectl -n istio-system get pods -l app=ztunnel -o wide +echo '--- waypoint pods ---' && kubectl -n zt-demo get pods -l gateway.networking.k8s.io/gateway-name=payments-waypoint -o wide +``` + +![Full ambient mesh pod listing](/img/blog/zero-trust-istio-sidecar-vs-ambient/16-full-ambient-view.png) + +The entire ambient mesh in one view. Every application pod sits at `1/1` READY. No sidecar anywhere. One ztunnel pod for the node. One `payments-waypoint` pod and only one because it's the only service that needed L7. This is the resource story ambient mode makes visible directly in a pod list rather than asserted in a comparison table. + +### The Nuance + +Getting the waypoint running was the easy part. The `AuthorizationPolicy` that worked perfectly in sidecar mode does not immediately start enforcing anything once the waypoint existed. + +In sidecar mode an `AuthorizationPolicy` attaches to a workload with a plain label selector (`selector: matchLabels: app: payments`) because the enforcement point (the sidecar) lives inside that exact pod. In ambient mode HTTP-level enforcement happens on the waypoint, a separate workload. The policy has to explicitly target that waypoint resource. + +**First attempt: `targetRefs` pointing at the `Gateway`:** + +```bash +kubectl apply -f - < 80/TCP 31m app=payments,istio.io/use-waypoint=payments-waypoint +``` + +Now the service carries the label `istio.io/use-waypoint=payments-waypoint`. Traffic to payments is routed through the waypoint. + +Istio's docs recommend `targetRefs: Service` as the more precise option because it binds the policy to the service abstraction rather than the proxy instance. In this demo I used `targetRefs: Gateway` because it feels intuitive: the waypoint is the actual enforcement point so targeting it directly makes the mechanics explicit. Both patterns work. The real gotcha we hit was the `use-waypoint` label on the Service. That is what routes traffic through the waypoint without it, neither Gateway targeting nor Service targeting would have enforced anything. If you are building this for production, use `targetRefs: Service`. It decouples your policy from waypoint lifecycle and reads more naturally: you are protecting the payments service, not the payments-waypoint proxy. + +```bash +# Test from orders +kubectl run curl-orders -n zt-demo --image=curlimages/curl --restart=Never \ + --overrides='{"spec":{"serviceAccountName":"orders"}}' \ + -- curl -s -o /dev/null -w '%{http_code}\n' \ + http://payments.zt-demo.svc.cluster.local/post -X POST -d '{"amount": 500}' +``` + +```text +200 +``` + +Orders gets a `200`. + +```bash +# Test from frontend +kubectl -n zt-demo exec deploy/frontend -- \ + curl -s -o /dev/null -w '%{http_code}\n' \ + http://payments/post -X POST -d '{"amount": 500}' +``` + +```text +403 +``` + +Frontend gets a `403`. Same cluster, same service, same everything except its identity. The gap is identical to sidecar mode. The mechanism underneath is completely different. + +--- + +## What Actually Changed, Side by Side + +| Aspect | Sidecar Mode | Ambient Mode | +|---|---|---| +| Joining the mesh | Full rollout restart of every deployment required | One label applied to already-running pods. Zero restarts. | +| Pod shape | Full Envoy proxy in every application pod (`2/2` READY) | No sidecar in application pods. A waypoint is deployed as its own separate pod, only for the service that needs L7 rules. | +| Policy authoring | `selector.matchLabels` targets the workload directly | `targetRefs` targets the Gateway or Service, plus an `istio.io/use-waypoint` label on the Service | +| Fail-safe when policy exceeds L4 | Not applicable, every pod has a full proxy | ztunnel accepts the policy without erroring, but fails safe to DENY for HTTP attributes it can't evaluate. The AuthorizationPolicy status field explains why. | +| mTLS enforcement | Configured via `PeerAuthentication` | Active by default for in-mesh traffic. `PeerAuthentication` is optional, for explicit control. | + +### What Istio's Own Comparison Publishes + +Worth citing: Istio reports typical p90/p99 latency of roughly **0.6 to 0.9ms per hop** in sidecar mode since both the source and destination sidecar process every request versus roughly **0.15 to 0.2ms with ztunnel alone** and **0.4 to 0.5ms when a waypoint is in the path**. That's Istio's benchmark in their environment, not mine. Worth verifying on your own hardware and environment. + +--- + +## Why Run This Yourself Instead of Reading a Comparison Table + +Every sidecar-vs-ambient article can list the theoretical differences in a table. What a table can't do is show you the exact moment ztunnel refuses your policy and tells you why or make you feel the difference between watching four pods restart and watching a label apply to four pods that never blinked. That gap between reading and running is the actual reason this exists as a runnable project instead of another explainer. Clone it, break it and the L4/L7 split stops being a diagram and starts being something you've debugged. + +--- + +## What's Next + +If you want to take this further here are immediate hands-on next steps that extend the core comparison: + +1. **Verify the fail-safe yourself.** Delete the `payments-waypoint` Gateway but keep the L7 `AuthorizationPolicy` applied. Confirm that all traffic to payments is denied. Then recreate the waypoint, re-apply the Service label and watch access restore. This proves the architecture is protecting you from misconfiguration. +2. **Try `targetRefs: Service` vs `Gateway`.** We used `kind: Gateway` in this demo. Try switching to `kind: Service` (group: `""`, name: `payments`) and confirm identical behavior. Understand when each approach is more appropriate. +3. **Add a second waypoint.** Give `inventory` its own waypoint and an L7 policy. Show that waypoints are per-service not per-namespace and that you only pay the L7 proxy cost where you actually need it. +4. **Measure latency with Fortio.** Run a formal benchmark pass against both modes on the same hardware with Prometheus and Grafana for dashboards to verify Istio's published figures with your own first-hand measurements on a replicable environment. + +--- + +Repository: `github.com/Prianshu-git/Service-mesh-Zero-Trust-migration` + +--- + +# Running Qwen3.8-Flash-Next on a DGX Spark and RTX PRO 6000 + +- Canonical: https://blog.kubesimplify.com/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000 +- Published: 2026-08-27 + +Qwen dropped Qwen3.8-Flash-Next this week, and the first thing I saw on my timeline was somebody saying it will not fit on a single DGX Spark. The NVFP4 weights are around 135 GB, a Spark has 128 GB of unified memory, so you need two of them. + +That is correct. I checked it and I will show you why. But it is also only part of the story, because there is one build of this model that does fit on a single Spark, and the reason it fits turned out to be more interesting than the fitting. + +I have a DGX Spark and access to a box with 8 RTX PRO 6000 Blackwell cards, so let's run it on both and see what the numbers actually look like. + +In this post we will go through: + +- What Qwen3.8-Flash-Next actually is, and why its size is confusing +- Why "NVFP4 is 135 GB" and "the GGUF is 67 GB" are both true for the same model +- Getting it running on a single DGX Spark with llama.cpp +- Getting it running on RTX PRO 6000 with vLLM, and how it scales across 1, 2 and 4 GPUs +- Why two GPUs beat four on this hardware + +Every number in this post was measured on my own machines. Where I quote somebody else's number, I say so. + +## What the model is + +Qwen3.8-Flash-Next is a mixture-of-experts model. Total parameters are 176.94B, and that splits into two very different halves: + +- **125B in the model proper**, of which 512 experts do most of the work. For any given token the router picks only 10 experts plus 1 shared expert. +- **51B in an N-gram embedding table**, which is a giant lookup table rather than something you do maths with. + +Qwen puts the active parameters at about 6B per token. That is the whole point of the design: you get the knowledge of a very large model while paying the compute bill of a small one. Worth noting llama.cpp labels the same model `A3B`, so the two are counting slightly different things, and I have not dug into which is right. + +The attention is a hybrid. Three out of every four layers use Gated DeltaNet, which compresses the history into a fixed-size state, and every fourth layer uses Qwen Sparse Attention (QSA), which looks at the full context but only scores it in compressed blocks. Qwen calls this a preview of the Qwen4 architecture, and the model type in `config.json` is literally `qwen4_exp`. + +## Why the size question is confusing + +Here is where I lost an hour, so let me save you the same trouble. + +You would assume "NVFP4" means the whole model is squeezed into 4 bits. It does not. I opened `quantization_config` in both official checkpoints, and both have a `modules_to_not_convert` list. Only the **routed experts** get quantized. Attention, GDN, QSA, shared experts, routers, `lm_head`, embeddings, the vision encoder and the MTP head all stay in BF16. + +The routed experts are 120.8B of the 125B, so that still covers most of the model. But the 51B N-gram table is the problem. It is stored as FP8 and expanded to BF16 when loaded, which is about 102 GB sitting in memory. + +That is why the four builds are so far apart in size: + +| Build | Size on disk | Fits one Spark (121 GiB usable)? | +| --- | --- | --- | +| BF16 | 335.3 GiB | No | +| FP8 | 172.8 GiB | No | +| NVFP4 | 135.3 GB | No | +| GGUF `UD-IQ1_S` | 67.55 GiB | **Yes** | + +The GGUF is the only build that quantizes the N-gram table too. That is the entire reason it fits. + +Now, vLLM has a flag called `VLLM_PLE_CPU_OFFLOAD=1` that pushes that table into host RAM. And this is not a hack somebody bolted on. The Qwen tech report says the tables are "held off the accelerator", and they placed the N-gram layer at **layer 2 specifically so that fetching from host memory overlaps with the compute of layer 1**. The architecture was designed for the table to live somewhere else. + +Which is also why that flag does nothing on a Spark. On a Spark, host RAM *is* the same unified pool as GPU memory. There is nowhere to offload to. + +## Test environment + +| | DGX Spark | RTX PRO 6000 box | +| --- | --- | --- | +| GPU | 1x GB10, 128 GB unified (124610 MiB visible to CUDA) | 8x RTX PRO 6000 Blackwell Server Edition, 97887 MiB each | +| Compute capability | 12.1 | 12.0 | +| Driver | 580.159.03 | 610.43.02 | +| Host RAM | shared with GPU | 1259 GB | +| GPU interconnect | n/a | **No NVLink**, every pair reports `SYS` | +| Engine | llama.cpp build 30, commit `035e227` | vLLM, image `vllm/vllm-openai:qwen38-flash-next` | +| Model build | `unsloth/Qwen3.8-Flash-Next-GGUF` UD-IQ1_S | `Qwen/Qwen3.8-Flash-Next-FP8` | + +On the RTX box only 4 of the 8 cards were free, so everything below uses GPUs 1, 4, 5 and 6. + +## Part 1: the DGX Spark + +### llama.cpp support is not merged yet + +First problem. My existing llama.cpp knows `QWEN3NEXT` but not `qwen4_exp`, so it simply will not load this model. Support is an open pull request, [#27742](https://github.com/ggml-org/llama.cpp/pull/27742), written by [Daniel Han](https://github.com/danielhanchen) of [Unsloth](https://unsloth.ai) - all 33 commits of it. Converter, text graph, sparse attention, vision and three quantizer fixes. The entire Spark half of this post exists because of that PR. + +So we build it: + +```bash +export PATH=$PATH:/usr/local/cuda/bin +git clone --depth 30 --branch qwen4exp/qwen3.8-flash-next \ + https://github.com/unslothai/llama.cpp.git ~/llama-qwen4exp +cd ~/llama-qwen4exp +cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES=121 \ + -DGGML_CUDA_FA=ON -DCMAKE_BUILD_TYPE=Release +cmake --build build --config Release -j 16 \ + --target llama-server llama-cli llama-bench llama-perplexity +``` + +`121` is the GB10 compute capability. Check it worked: + +``` +$ ~/llama-qwen4exp/build/bin/llama-cli --version +version: 0.3.0-dev (build 30, commit 035e227) +built with GNU 13.3.0 for Linux aarch64 +``` + +### Getting the weights + +```bash +hf download unsloth/Qwen3.8-Flash-Next-GGUF --local-dir ~/qwen38/gguf +``` + +If that stalls at 0 B/s, it is the Xet transport. Set `HF_HUB_DISABLE_XET=1` and keep the worker count at 6 to 8. I tried 24 workers and got `SSL handshake timed out`. + +### Running it + +```bash +~/llama-qwen4exp/build/bin/llama-server \ + -m ~/qwen38/gguf/UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00001-of-00003.gguf \ + -ngl 999 -c 16384 --host 127.0.0.1 --port 8099 --jinja +``` + +It loads in about 30 seconds and sits at 72.5 GiB of the 121 GiB available. That leaves roughly 49 GiB free, which is a lot more headroom than I expected. + +Here is llama-bench, three repetitions: + +```bash +~/llama-qwen4exp/build/bin/llama-bench \ + -m ~/qwen38/gguf/UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00001-of-00003.gguf \ + -ngl 999 -p 2048,8192,32768 -n 128 -r 3 +``` + +``` +| model | size | params | backend | ngl | test | t/s | +| qwen4exp A3B IQ1_S | 67.55 GiB | 176.94 B | CUDA | 999 | pp2048 | 797.76 ± 2.08 | +| qwen4exp A3B IQ1_S | 67.55 GiB | 176.94 B | CUDA | 999 | pp8192 | 747.53 ± 3.24 | +| qwen4exp A3B IQ1_S | 67.55 GiB | 176.94 B | CUDA | 999 | pp32768 | 599.65 ± 1.18 | +| qwen4exp A3B IQ1_S | 67.55 GiB | 176.94 B | CUDA | 999 | tg128 | 34.54 ± 0.18 | +``` + +**34.5 tokens per second on a single Spark, for a model with 176.94B parameters.** + +I did not believe that at first either, so let's sanity check it two ways. + +First against the hardware. The GB10 is specified at about 273 GB/s of memory bandwidth (that is the spec sheet, not something I measured). Decoding reads roughly 5.37 GB per token here, because only 2.36B of the 120.8B expert parameters are touched for any given token. That puts the ceiling around 50 tok/s, and we measured 34.5, or 68% of it. Comfortably under the roof, which is where a real measurement should sit. + +Second against my own earlier numbers. When I [benchmarked the dense Qwen3.8-27B on this same Spark](https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark) a couple of weeks ago, llama.cpp gave 11.6 tok/s. A sparse 177B model running about three times faster than a dense 27B one is what you would expect when only a small slice is active per token. + +### The prefill curve is the interesting bit + +Look again at those prefill numbers. Going from 2,048 tokens to 32,768 tokens is 16 times the context, and throughput only drops 25%. That flatness is consistent with QSA doing its job, although I should be honest that I did not run a dense-attention ablation to prove QSA is the cause. + +### "IQ1_S" is not a 1-bit model + +The quant is called `UD-IQ1_S` and llama.cpp reports `IQ1_S - 1.5625 bpw`, which makes it sound like a 1-bit model. So I dumped the actual tensor types in the file: + +| Type | Size | Share | +| --- | --- | --- | +| IQ4_NL | 47.92 GiB | 70.9% | +| IQ1_S | 10.38 GiB | 15.4% | +| IQ2_XXS | 5.64 GiB | 8.3% | +| Q5_K, Q8_0, Q4_K, Q6_K, F32, BF16 | 3.63 GiB | 5.4% | + +**Effective 3.28 bits per weight, not 1.56.** Seventy percent of the bytes are ordinary 4-bit. Unsloth's dynamic quants spend the bit budget where it matters and squeeze the rest, and the biggest thing getting squeezed is that 51B lookup table. + +### What it costs you + +A couple of prompts coming back correct is not evidence that a quant is fine, so let's measure it with perplexity. + +Quick definition, because the name is confusing and there is now a search company called Perplexity that has nothing to do with this. **Perplexity scores how surprised a model is by text it has never seen.** You feed it real writing and at each word check what probability it gave to the word that actually came next, then boil that down to roughly "how many words was it torn between at each step". **Lower is better.** It runs locally, no API involved. + +```bash +# llama.cpp's own scripts/get-wikitext-2.sh is broken: it does not follow the +# S3 redirect and leaves you with a 467-byte XML error instead of a zip. +curl -sL -o /tmp/wt2.zip \ + "https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip" +unzip -oq /tmp/wt2.zip -d /tmp/ + +~/llama-qwen4exp/build/bin/llama-perplexity \ + -m ~/qwen38/gguf/UD-IQ1_S/Qwen3.8-Flash-Next-UD-IQ1_S-00001-of-00003.gguf \ + -f /tmp/wikitext-2-raw/wiki.test.raw -ngl 999 -c 2048 +``` + +``` +Final estimate: PPL = 4.7876 +/- 0.02848 +``` + +That is wikitext-2, 145 chunks at context 2048. Daniel Han reports 4.0068 for llama.cpp at high precision and 4.0126 for the reference implementation in the PR write-up. Those are **his** numbers, not mine, and I could not reproduce them because no higher-precision GGUF of this model has been published yet. + +Taking his figure at face value, this quant costs roughly 19% higher perplexity. A normal Q4_K_M usually costs 1 to 3%. So it is a real trade, not a free lunch. It answers questions correctly in casual use, and I would still not reach for it when accuracy matters. + +## Part 2: RTX PRO 6000 with vLLM + +vLLM had day-zero support with a dedicated image, so this side was much less work than the Spark. Getting the 185 GB checkpoint down was the slow part: + +```bash +docker pull vllm/vllm-openai:qwen38-flash-next +HF_HUB_DISABLE_XET=1 hf download Qwen/Qwen3.8-Flash-Next-FP8 --local-dir /llm/qwen38/fp8 +``` + +HuggingFace crawled from this box, so I pulled it from ModelScope instead, which serves the identical 145-file manifest. + +```bash +docker run -d --name q38-tp2 --gpus '"device=1,4"' --ipc=host --shm-size=32g \ + -v /llm/qwen38:/llm/qwen38 -e VLLM_PLE_CPU_OFFLOAD=1 -p 8010:8000 \ + vllm/vllm-openai:qwen38-flash-next \ + --model /llm/qwen38/fp8 --served-model-name q38 \ + --tensor-parallel-size 2 --gpu-memory-utilization 0.90 \ + --max-model-len 32768 --max-num-seqs 32 \ + --enable-prefix-caching --no-enable-flashinfer-autotune \ + --reasoning-parser qwen3 +``` + +If you leave out `--reasoning-parser qwen3`, the model's thinking text ends up inside the normal reply content. Ask for it. + +For the TP4 runs it is the same command with `--gpus '"device=1,4,5,6"'` and +`--tensor-parallel-size 4`. To compare against keeping the N-gram table on the GPU, set +`-e VLLM_PLE_CPU_OFFLOAD=0`. For speculative decoding, append the MTP config shown later. + +Every config below was benchmarked with exactly the same command, only `$C` changing: + +```bash +docker exec q38-tp2 vllm bench serve \ + --backend openai-chat --model /llm/qwen38/fp8 --served-model-name q38 \ + --endpoint /v1/chat/completions --base-url http://localhost:8000 \ + --dataset-name random --random-input-len 1024 --random-output-len 512 \ + --max-concurrency $C --num-prompts $((C*4)) --ignore-eos +``` + +Before benchmarking anything I asked it a question with a known answer, because a healthy `/health` endpoint does not mean the model is producing sense. It got "a train leaves at 14:35 and arrives at 21:10 the next day" right at 30 hours 35 minutes, so we are good. + +### How many GPUs do you need? + +`--tensor-parallel-size` (TP) is how many GPUs each layer's weight matrices are sliced across. Not "layer 1 on this GPU, layer 2 on that one", that is pipeline parallelism. TP cuts every matrix into pieces, so each GPU computes a partial answer and then they all swap and add. That swap is an all-reduce and it happens at every layer, for every token. + +With the N-gram table offloaded to host RAM, the weights need about 123 GiB on the GPU, and each card has 95.6 GiB. So one card should not be enough. It is not: + +``` +torch.OutOfMemoryError: CUDA out of memory. GPU 0 has a total capacity of +95.01 GiB of which 210.38 MiB is free ... 94.02 GiB is allocated by PyTorch +``` + +TP3, by the way, is not an option at all. I assumed this was about the 2 KV heads on the attention layers, so I tried it to be sure, and the real reason is different: + +``` +AssertionError: 16 is not divisible by 3 +``` + +The 16 is `linear_num_key_heads`, the key heads in the Gated DeltaNet layers, and those are 36 of the 48 layers. Your TP size has to divide 16, so the usable values are 1, 2, 4, 8 and 16. + +Here is TP2 and TP4, benchmarked with 1024 input and 512 output tokens: + +| Config | 1 stream | 32 streams | Median TPOT, 1 stream | KV cache | +| --- | --- | --- | --- | --- | +| TP1 | out of memory | - | - | - | +| TP2 | **81.45 tok/s** | 739.06 tok/s | 10.01 ms | 19.6 GiB | +| TP4 | 64.61 tok/s | **805.38 tok/s** | 13.85 ms | 48.74 GiB | + +**Two GPUs are 26% faster than four for a single user.** That surprised me until I looked at the wiring. This box has no NVLink, and `nvidia-smi topo -m` reports every GPU pair as `SYS`, meaning traffic crosses PCIe and the CPU sockets. Now remember only 6B parameters are active per token, so there is barely any maths to divide up. Splitting a tiny job across more GPUs mostly means paying more postage. The extra cards still earn their keep under load, where the bigger KV cache lets you batch 32 users and win on total throughput. + +The practical rule: use the smallest TP that fits in memory, and only go wider when you need the KV cache for longer context or more users. + +There is also no pipeline-parallel escape hatch here. The vLLM recipe states the N-gram embedding does not support pipeline parallelism, so on a box with bad interconnect you cannot fall back to PP the way you normally would. + +### What does offloading the N-gram table actually cost? + +The tech report implies host prefetching is nearly free. On this hardware it is cheap but not free: + +| TP4 config | 1 stream | 32 streams | KV cache | +| --- | --- | --- | --- | +| N-gram table on GPU | 74.84 tok/s | 772.16 tok/s | 4.7 GiB | +| N-gram table on host | 64.61 tok/s | 805.38 tok/s | 48.74 GiB | + +Keeping the table on the GPU is about 16% faster for one user, because you skip the round trip over PCIe. But it eats the memory your KV cache wanted, and it collapses from 48.74 GiB to 4.7 GiB. Maximum concurrency drops from 74x to 10x. + +Unless you are serving exactly one person, offload it. + +### Speculative decoding, and why benchmark workload decides the answer + +The model ships an MTP head, so let's turn it on: + +```bash +--speculative-config '{"method":"mtp","num_speculative_tokens":3}' +``` + +Measured with the same `vllm bench serve` command as everything above, which uses +synthetic random tokens: + +| TP4 config | 1 stream | 32 streams | Median TTFT at 32 | +| --- | --- | --- | --- | +| without MTP | 64.61 tok/s | 805.38 tok/s | 2578 ms | +| with MTP | **87.87 tok/s** | 693.72 tok/s | **588 ms** | + +That reads as 36% faster for one user, at the cost of 14% of peak throughput. I nearly +left it there. Then someone asked what the acceptance rate was, which is the number that +actually decides whether speculative decoding is worth anything, and I had not measured it. + +vLLM exposes it. Here is what the counters say: + +| workload | acceptance | mean accepted length | +| --- | --- | --- | +| synthetic random, 1024 in / 512 out | 71.3% | 3.14 of max 4 | +| synthetic random, 512 in / 256 out | 84.4% | 3.53 of max 4 | +| **real code and prose prompts, greedy** | **55.3%** | **2.66 of max 4** | + +Real prompts accept considerably worse than random ones. That is the opposite of what I +expected, and the reason is worth knowing if you benchmark anything: `--dataset-name random` +feeds the model random token IDs. Given nonsense, it produces repetitive low-entropy text, +and a draft head predicts repetitive text very easily. **Random-token benchmarks flatter +speculative decoding.** + +So I re-ran on five genuine prompts, an LRU cache in Python, a Kubernetes explanation, a Go +CSV reader, a bash one-liner and a plain-English TP explainer, greedy decoding, single +stream, measuring wall clock: + +| TP4, real prompts, temp 0 | tok/s | +| --- | --- | +| without MTP | 49.90 | +| with MTP | **124.93** | + +**2.5x.** Far better than the 36% the synthetic benchmark implied, despite the lower +acceptance rate. Note this is a different measurement method from the table above, wall +clock across whole requests rather than vLLM's output-token throughput, so compare within +each table and not across them. + +The lesson is not that one number is right and the other wrong. Both are real. It is that +a speculative decoding result without its workload and its acceptance rate does not tell +you anything you can act on. + +### Does MTP change the answer? + +Speculative decoding is supposed to be lossless. The draft head proposes, the full model +verifies, and rejected tokens are discarded, so the output distribution should be +untouched. Worth checking rather than trusting. + +Same prompt, temperature 0, seed 42, five runs each, on the same four GPUs with the same +checkpoint, MTP the only variable: + +``` +MTP off : 3575aff8aa9c1df5 x5 +MTP on : 3575aff8aa9c1df5 x5 +``` + +Byte-identical, and identical to each other. Speculative decoding here costs you nothing in +output fidelity. Worth knowing if you cache responses or snapshot-test them. + +For reference, llama.cpp on the Spark is also fully deterministic across five runs, though +that is a different box and a different quant so the hashes are not comparable. + +One caveat on all of this: single stream. Under continuous batching, vLLM's batch +composition varies between runs and that is where reproducibility usually breaks, not from +MTP. + +### One card, with NVFP4 + +There is a community NVFP4 build from RadixArk. It is documented for SGLang, but vLLM picked it up anyway (`Detected ModelOpt NVFP4 checkpoint`): + +```bash +docker run -d --name q38-nvfp4-tp1 --gpus '"device=1"' --ipc=host --shm-size=32g \ + -v /llm/qwen38:/llm/qwen38 -e VLLM_PLE_CPU_OFFLOAD=1 -p 8012:8000 \ + vllm/vllm-openai:qwen38-flash-next \ + --model /llm/qwen38/nvfp4 --served-model-name q38 \ + --tensor-parallel-size 1 --gpu-memory-utilization 0.93 \ + --max-model-len 16384 --max-num-seqs 16 \ + --no-enable-flashinfer-autotune --reasoning-parser qwen3 +``` + +The weights genuinely fit on a single card: + +``` +Actual usage is 74.75 GiB for consumed memory (weights + non-torch), +1.85 GiB for peak activation, and 0.28 GiB for CUDAGraph memory. +Current kv cache memory in use is 11.76 GiB. +``` + +74.75 GiB of weights on one 95.6 GiB card, with 11.76 GiB of KV cache left over. So the memory answer is yes. + +I cannot give you a speed number though. The engine finished loading, captured its CUDA graphs at 06:34:14, and then the API server never came up. Twenty minutes later `Application startup complete` had still not been printed a single time, `/health` was refusing connections, and one CPU core was spinning. The only errors in the log were harmless transformers docstring warnings. RadixArk documented this checkpoint for SGLang and not vLLM, so I am not shocked, but I am not going to invent a number I did not measure. + +## Both machines side by side + +![Qwen3.8-Flash-Next benchmarks on DGX Spark and RTX PRO 6000](/img/blog/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000/benchmarks-both-machines.png) + +| | DGX Spark | RTX PRO 6000 | +| --- | --- | --- | +| Build | GGUF UD-IQ1_S, 3.28 bpw | FP8, 172.8 GiB | +| Engine | llama.cpp (unmerged PR) | vLLM (day-zero support) | +| GPUs used | 1 | 2 or 4 | +| Best single stream | 34.5 tok/s | 87.9 tok/s (TP4 + MTP) | +| Best throughput | not measured | 805 tok/s at 32 streams | +| Memory | 72.5 of 121 GiB | ~66 GiB per GPU at TP2 | + +These are not really competing. One is a desktop box running a heavily compressed build, the other is four datacenter cards running the full FP8 checkpoint. What I find genuinely interesting is that the gap is only about 2.5x. + +## What I did not measure + +To be straight about the edges of this post: + +- **No high-precision perplexity baseline.** The 4.0068 and 4.0126 figures are the PR author's, and no higher-precision GGUF exists yet for me to check them against. +- **No proof that QSA causes the flat prefill curve.** It is consistent with the architecture, but I ran no ablation. +- **No NVFP4 throughput.** The weights fit on one card, the server never came up. +- **No concurrency sweep on the Spark.** llama-bench numbers there are single stream. +- **No BF16 run anywhere.** At 335 GiB it was not worth the download. + +## Thanks + +Two things made the Spark side of this possible and both came from the same place. + +[Daniel Han](https://x.com/danielhanchen) at [Unsloth](https://x.com/UnslothAI) wrote the +llama.cpp support in PR [#27742](https://github.com/ggml-org/llama.cpp/pull/27742), every +commit of it, within a day of the model landing. He also published the dynamic quant that +is the only build small enough to fit on one Spark, and his write-up of the PR includes +perplexity and top-1 agreement numbers against the reference implementation, which is what +let me sanity check my own. That is a lot of careful work given away for free. + +Thanks also to the [llama.cpp](https://github.com/ggml-org/llama.cpp) maintainers, and to +[RadixArk](https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4) for the NVFP4 +conversion, which fit on a single RTX PRO 6000 even though I could not get it serving. + +## Wrapping up + +The claim that started this was right: NVFP4 does not fit on one DGX Spark. But a single Spark still runs this 177B model at 34.5 tok/s through llama.cpp, because Unsloth's GGUF is the one build that also compresses the 51B N-gram table, and it costs you about 19% perplexity to do it. + +On the RTX PRO 6000 box the surprise was that two GPUs beat four for a single user. If you are sizing hardware for sparse MoE models, more cards past the point where the weights fit will buy you batch throughput and KV cache, not lower latency, especially without NVLink. + +The scripts, recipes and raw benchmark output are in the repo if you want to reproduce any of this. If you run it on different hardware I would love to see your numbers, so send them over on X [@SaiyamPathak](https://x.com/SaiyamPathak). + +--- + # The Local LLM Glossary: Every Term, Flag, and Number in Plain English - Canonical: https://blog.kubesimplify.com/local-llm-glossary diff --git a/public/llms.txt b/public/llms.txt index 6eae55e4c..cc9d8bef5 100644 --- a/public/llms.txt +++ b/public/llms.txt @@ -4,7 +4,7 @@ ## About -Kubesimplify is a community-driven publication on cloud-native technologies, with 198 in-depth technical articles by 62 practitioner authors. We cover Kubernetes (kubelet internals, scheduling, networking, operators), container runtimes (containerd, CRI-O, Docker), GitOps (Argo CD, Flux), service meshes, observability, AI/ML infrastructure on Kubernetes, GPU workloads, platform engineering, and the broader CNCF ecosystem. +Kubesimplify is a community-driven publication on cloud-native technologies, with 201 in-depth technical articles by 63 practitioner authors. We cover Kubernetes (kubelet internals, scheduling, networking, operators), container runtimes (containerd, CRI-O, Docker), GitOps (Argo CD, Flux), service meshes, observability, AI/ML infrastructure on Kubernetes, GPU workloads, platform engineering, and the broader CNCF ecosystem. Authoritative, practitioner-written, citation-friendly. Articles include code examples, diagrams, and references. @@ -34,8 +34,11 @@ Authoritative, practitioner-written, citation-friendly. Articles include code ex - Cloud Native Security: https://blog.kubesimplify.com/hub/security (network policies, Falco, Kyverno, SLSA supply-chain) - Linux Fundamentals: https://blog.kubesimplify.com/hub/linux (shell, sysadmin, networking primitives) -## Recent posts (most recent 30 of 198) +## Recent posts (most recent 30 of 201) +- [Running a big LLM across multiple GPUs with vLLM](https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm) (2026-09-01). A runbook for serving a model too big for one GPU: download to serving in seven steps, with every vLLM flag, startup log line and real error explained, plus tensor, pipeline and expert parallelism benchmarked head to head on a 235B model across four RTX PRO 6000 cards. +- [Zero Trust in Practice: Migrating from Istio Sidecar to Ambient Mode](https://blog.kubesimplify.com/zero-trust-istio-sidecar-vs-ambient) (2026-08-31). A hands-on comparison of Istio sidecar and ambient mode for zero-trust service mesh. Same app, same policy, two architectures proven step by step on a local cluster. +- [Running Qwen3.8-Flash-Next on a DGX Spark and RTX PRO 6000](https://blog.kubesimplify.com/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000) (2026-08-27) - [The Local LLM Glossary: Every Term, Flag, and Number in Plain English](https://blog.kubesimplify.com/local-llm-glossary) (2026-08-18). Plain-English definitions for every term you hit in local LLM posts: prefill and decode, tokens per second, FP8 and NVFP4, Q4_K_M, KV cache, YaRN, Gated DeltaNet, speculative decoding, and every vLLM, llama.cpp, and Ollama flag worth knowing. - [Running Qwen3.8-27B on DGX Spark](https://blog.kubesimplify.com/qwen3-8-27b-on-dgx-spark) (2026-08-17). Qwen3.8-27B on DGX Spark with llama.cpp, Ollama, vLLM, and SGLang: the recipes, the tokens per second I measured, MTP speculative decoding, and the sharp edges I hit along the way. - [I Ran an AI SRE Copilot on My Own Hardware. Here Is What It Actually Does.](https://blog.kubesimplify.com/nudgebee-ai-sre-copilot-hands-on) (2026-08-17). Running NudgeBee v1.4.0 end to end - a self-hosted AIOps platform behind AI-SRE, AI-FinOps, AI-K8sOps, and agentic automation - on a Mac, a kiac cluster, and a DGX Spark. @@ -63,21 +66,18 @@ Authoritative, practitioner-written, citation-friendly. Articles include code ex - [Day 6: Run an LLM on Your Laptop - With Docker](https://blog.kubesimplify.com/day-6-run-an-llm-on-your-laptop-with-docker) (2026-04-30). \"Pull AI models from Docker Hub, run them locally with GPU acceleration, and build an AI-powered app - [A Kubeconfig for GKE That Doesn't Need gcloud](https://blog.kubesimplify.com/a-kubeconfig-for-gke-that-doesnt-need-gcloud) (2026-04-29) - [Day 5: Docker Compose - How Docker Actually Gets Used](https://blog.kubesimplify.com/day-5-docker-compose-how-docker-actually-gets-used) (2026-04-28) -- [What Actually Happens When kube-scheduler Picks a Node (13 Stages Inside Kubernetes)](https://blog.kubesimplify.com/kube-scheduler-deep-dive) (2026-04-28). How kube-scheduler picks a node: 13 framework stages, 14 Filter plugins, 9 Score plugins, live preemption demo. -- [Day 4: Breaking Isolation on Purpose - Volumes, Networks, and the Real World](https://blog.kubesimplify.com/day-4-breaking-isolation-on-purpose-volumes-networks-and-the-real-world) (2026-04-27) -- [Day 3: Stop Writing Dockerfiles From Scratch](https://blog.kubesimplify.com/day-3-stop-writing-dockerfiles-from-scratch) (2026-04-24). Stop writing Dockerfiles from scratch. A Docker Captain walks through docker init, layer caching, multi-stage builds, and docker debug for 2026. ## Topics covered (auto-derived from tags) -- kubernetes (100 articles): https://blog.kubesimplify.com/tag/kubernetes +- kubernetes (101 articles): https://blog.kubesimplify.com/tag/kubernetes - devops (71 articles): https://blog.kubesimplify.com/tag/devops - docker (31 articles): https://blog.kubesimplify.com/tag/docker - k8s (27 articles): https://blog.kubesimplify.com/tag/k8s - linux (19 articles): https://blog.kubesimplify.com/tag/linux - containers (17 articles): https://blog.kubesimplify.com/tag/containers - cloud (16 articles): https://blog.kubesimplify.com/tag/cloud -- nvidia (14 articles): https://blog.kubesimplify.com/tag/nvidia -- llm (12 articles): https://blog.kubesimplify.com/tag/llm +- nvidia (15 articles): https://blog.kubesimplify.com/tag/nvidia +- llm (14 articles): https://blog.kubesimplify.com/tag/llm - aws (12 articles): https://blog.kubesimplify.com/tag/aws - cloud-native (11 articles): https://blog.kubesimplify.com/tag/cloud-native - security (11 articles): https://blog.kubesimplify.com/tag/security @@ -85,25 +85,25 @@ Authoritative, practitioner-written, citation-friendly. Articles include code ex - go (9 articles): https://blog.kubesimplify.com/tag/go - git (9 articles): https://blog.kubesimplify.com/tag/git - linux-for-beginners (9 articles): https://blog.kubesimplify.com/tag/linux-for-beginners +- platform-engineering (8 articles): https://blog.kubesimplify.com/tag/platform-engineering +- ai (8 articles): https://blog.kubesimplify.com/tag/ai - local-ai (8 articles): https://blog.kubesimplify.com/tag/local-ai - github (8 articles): https://blog.kubesimplify.com/tag/github - terraform (8 articles): https://blog.kubesimplify.com/tag/terraform -- ai (7 articles): https://blog.kubesimplify.com/tag/ai -- platform-engineering (7 articles): https://blog.kubesimplify.com/tag/platform-engineering +- gpu (7 articles): https://blog.kubesimplify.com/tag/gpu - docker-images (7 articles): https://blog.kubesimplify.com/tag/docker-images - kubesimplify (7 articles): https://blog.kubesimplify.com/tag/kubesimplify - linux-basics (7 articles): https://blog.kubesimplify.com/tag/linux-basics -- ollama (6 articles): https://blog.kubesimplify.com/tag/ollama ## Top contributors -- [Saiyam Pathak](https://blog.kubesimplify.com/author/saiyam-pathak) (40 posts) +- [Saiyam Pathak](https://blog.kubesimplify.com/author/saiyam-pathak) (41 posts) - [Saloni Narang](https://blog.kubesimplify.com/author/saloni-narang) (24 posts) - [Kunal Verma](https://blog.kubesimplify.com/author/kunal-verma) (12 posts) - [Dipankar Das](https://blog.kubesimplify.com/author/dipankar-das) (9 posts) - [Anurag Kumar](https://blog.kubesimplify.com/author/anurag-kumar) (8 posts) +- [Shubham Katara](https://blog.kubesimplify.com/author/shubham-katara) (6 posts) - [sysxplore](https://blog.kubesimplify.com/author/sysxplore) (6 posts) -- [Shubham Katara](https://blog.kubesimplify.com/author/shubham-katara) (5 posts) - [Arnav Barman](https://blog.kubesimplify.com/author/arnav-barman) (5 posts) - [Srinivas Karnati](https://blog.kubesimplify.com/author/srinivas-karnati) (4 posts) - [Barkatul Mujauddin](https://blog.kubesimplify.com/author/barkatul-mujauddin) (4 posts) diff --git a/public/rss.xml b/public/rss.xml index 2300c9b4a..beb1920cb 100644 --- a/public/rss.xml +++ b/public/rss.xml @@ -6,8 +6,32 @@ Deep dives on Kubernetes, AI infrastructure, GitOps, and the cloud-native stack, written by practitioners. en-us - Tue, 18 Aug 2026 09:00:00 GMT + Tue, 01 Sep 2026 10:00:00 GMT Kubesimplify static blog + + Running a big LLM across multiple GPUs with vLLM + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm + Tue, 01 Sep 2026 10:00:00 GMT + A runbook for serving a model too big for one GPU: download to serving in seven steps, with every vLLM flag, startup log line and real error explained, plus tensor, pipeline and expert parallelism benchmarked head to head on a 235B model across four RTX PRO 6000 cards. + vllmgpunvidiallmplatform-engineering + + + Zero Trust in Practice: Migrating from Istio Sidecar to Ambient Mode + https://blog.kubesimplify.com/zero-trust-istio-sidecar-vs-ambient + https://blog.kubesimplify.com/zero-trust-istio-sidecar-vs-ambient + Mon, 31 Aug 2026 10:00:00 GMT + A hands-on comparison of Istio sidecar and ambient mode for zero-trust service mesh. Same app, same policy, two architectures proven step by step on a local cluster. + istioservice-meshzero-trustkubernetesambient-mesh + + + Running Qwen3.8-Flash-Next on a DGX Spark and RTX PRO 6000 + https://blog.kubesimplify.com/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000 + https://blog.kubesimplify.com/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000 + Thu, 27 Aug 2026 06:30:00 GMT + + aillmvllmllamacppgpu + The Local LLM Glossary: Every Term, Flag, and Number in Plain English https://blog.kubesimplify.com/local-llm-glossary diff --git a/scripts/gen-local-llm-glossary-cover.mjs b/scripts/gen-local-llm-glossary-cover.mjs index 803b42205..45c0ec60a 100644 --- a/scripts/gen-local-llm-glossary-cover.mjs +++ b/scripts/gen-local-llm-glossary-cover.mjs @@ -1,5 +1,5 @@ // Excalidraw-style cover for the local LLM glossary post. -// Sketch helpers shared with scripts/gen-two-gpu-vllm-cover.mjs. +// Sketch helpers shared with scripts/gen-multi-gpu-vllm-cover.mjs. import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; diff --git a/scripts/gen-multi-gpu-vllm-cover.mjs b/scripts/gen-multi-gpu-vllm-cover.mjs new file mode 100644 index 000000000..6c1be76af --- /dev/null +++ b/scripts/gen-multi-gpu-vllm-cover.mjs @@ -0,0 +1,228 @@ +// Excalidraw-style cover for the multi-GPU vLLM article. +// Sketch helpers shared with scripts/gen-hami-diagrams.mjs. +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +let seed = 42; +const random = () => { + seed = (seed * 16807) % 2147483647; + return seed / 2147483647; +}; +const jitter = (amount) => (random() - 0.5) * amount * 2; + +const COLORS = { + ink: '#172033', + muted: '#5c677d', + green: { stroke: '#5d8f00', fill: '#d8f5a2' }, + blue: { stroke: '#1971c2', fill: '#a5d8ff' }, + violet: { stroke: '#862e9c', fill: '#eebefa' }, + orange: { stroke: '#d9480f', fill: '#ffd8a8' }, + red: { stroke: '#c92a2a', fill: '#ffc9c9' }, + teal: { stroke: '#087f5b', fill: '#b2f2bb' }, + gray: { stroke: '#495057', fill: '#e9ecef' }, +}; + +const FONT = 'Chalkboard SE, Comic Sans MS, sans-serif'; + +function roughLine(x1, y1, x2, y2, amount = 1.8) { + const middleX = (x1 + x2) / 2 + jitter(amount * 1.5); + const middleY = (y1 + y2) / 2 + jitter(amount * 1.5); + return `M ${(x1 + jitter(amount)).toFixed(1)} ${(y1 + jitter(amount)).toFixed(1)} Q ${middleX.toFixed(1)} ${middleY.toFixed(1)} ${(x2 + jitter(amount)).toFixed(1)} ${(y2 + jitter(amount)).toFixed(1)}`; +} + +class Sketch { + constructor(width, height, background = '#ffffff') { + this.width = width; + this.height = height; + this.background = background; + this.parts = []; + this.defs = []; + this.clipId = 0; + } + + add(value) { + this.parts.push(value); + } + + rect(x, y, width, height, options = {}) { + const { + stroke = COLORS.ink, + fill, + strokeWidth = 2.4, + dashed = false, + hachure = true, + radius = 7, + } = options; + + if (fill) { + if (hachure) { + // Hatch lines are clipped in math rather than with an SVG clipPath so + // the file renders identically in renderers without clipPath support. + const hatch = []; + for (let offset = -height; offset < width; offset += 11) { + const tMin = Math.max(0, -offset / height); + const tMax = Math.min(1, (width - offset) / height); + if (tMax - tMin < 0.05) continue; + const x1 = x + offset + height * tMin; + const y1 = y + height - height * tMin; + const x2 = x + offset + height * tMax; + const y2 = y + height - height * tMax; + hatch.push(roughLine(x1, y1, x2, y2, 1)); + } + this.add(``); + } else { + this.add(``); + } + } + + const points = [[x, y], [x + width, y], [x + width, y + height], [x, y + height]]; + for (let pass = 0; pass < 2; pass += 1) { + const path = points.map((point, index) => { + const next = points[(index + 1) % points.length]; + return roughLine(point[0], point[1], next[0], next[1], pass === 0 ? 2 : 1.2); + }).join(' '); + this.add(``); + } + } + + line(x1, y1, x2, y2, options = {}) { + const { stroke = COLORS.ink, strokeWidth = 2.4, dashed = false } = options; + this.add(``); + } + + arrow(x1, y1, x2, y2, options = {}) { + const { stroke = COLORS.ink, strokeWidth = 2.6, dashed = false } = options; + this.line(x1, y1, x2, y2, { stroke, strokeWidth, dashed }); + const angle = Math.atan2(y2 - y1, x2 - x1); + const length = 14; + for (const offset of [Math.PI * 0.82, -Math.PI * 0.82]) { + this.line( + x2, + y2, + x2 + length * Math.cos(angle + offset), + y2 + length * Math.sin(angle + offset), + { stroke, strokeWidth } + ); + } + } + + text(x, y, value, options = {}) { + const { + size = 22, + color = COLORS.ink, + anchor = 'middle', + weight = 500, + family = FONT, + } = options; + const safe = String(value) + .replace(/&/g, '&') + .replace(//g, '>'); + this.add(`${safe}`); + } + + lines(x, y, values, options = {}) { + const lineHeight = (options.size || 22) * (options.lineHeight || 1.28); + values.forEach((value, index) => this.text(x, y + index * lineHeight, value, options)); + } + + save(path) { + const svg = ` +${this.defs.join('')} + +${this.parts.join('\n')} +`; + writeFileSync(path, svg); + } +} + +const output = process.argv[2] || '.'; +mkdirSync(output, { recursive: true }); + + +const W = 1200; +const H = 630; +const sketch = new Sketch(W, H, '#fdfdfb'); + +sketch.text(64, 82, 'One big model, four GPUs', { size: 50, weight: 800, anchor: 'start' }); +sketch.text(64, 119, 'how a 235B model is cut up so it fits, and what that costs', { + size: 22, + color: COLORS.muted, + anchor: 'start', +}); +sketch.line(64, 139, 760, 139, { stroke: COLORS.muted, strokeWidth: 1.6, dashed: true }); + +// ── left: the model does not fit on one card ────────────── +sketch.text(64, 186, 'ONE CARD', { size: 18, weight: 800, anchor: 'start', color: COLORS.red.stroke }); + +const bY = 206; +sketch.rect(64, bY, 210, 132, { stroke: COLORS.gray.stroke, fill: '#ffffff', hachure: false, dashed: true }); +sketch.text(169, bY + 30, '95 GiB', { size: 19, color: COLORS.muted }); +sketch.text(169, bY + 54, 'usable', { size: 15, color: COLORS.muted }); + +// overflowing weights bar +sketch.rect(78, bY + 72, 330, 46, { stroke: COLORS.red.stroke, fill: COLORS.red.fill }); +sketch.text(200, bY + 95, '236 GB of weights', { size: 19, weight: 800, color: COLORS.red.stroke }); + +sketch.text(64, bY + 164, '2.3x too big', { size: 26, weight: 800, anchor: 'start', color: COLORS.red.stroke }); +sketch.text(64, bY + 192, 'no flag fixes this', { size: 16, anchor: 'start', color: COLORS.muted }); + +// ── divider ─────────────────────────────────────────────── +sketch.line(452, 186, 452, 452, { stroke: COLORS.muted, strokeWidth: 1.6, dashed: true }); + +// ── right: four cards, each holds a quarter ─────────────── +sketch.text(516, 186, 'FOUR CARDS, --tensor-parallel-size 4', { + size: 18, + weight: 800, + anchor: 'start', + color: COLORS.teal.stroke, +}); + +const cw = 145; +const gap = 10; +const gY = 206; +const palette = [COLORS.blue, COLORS.green, COLORS.violet, COLORS.orange]; +[0, 1, 2, 3].forEach((gpu) => { + const x = 516 + gpu * (cw + gap); + const c = palette[gpu]; + sketch.rect(x, gY, cw, 132, { stroke: c.stroke, fill: c.fill }); + sketch.text(x + cw / 2, gY + 30, `GPU ${gpu}`, { size: 20, weight: 800, color: c.stroke }); + sketch.text(x + cw / 2, gY + 60, '59 GB', { size: 18, weight: 700 }); + sketch.text(x + cw / 2, gY + 84, 'weights', { size: 14, color: COLORS.muted }); + sketch.text(x + cw / 2, gY + 112, '16 of 64 heads', { size: 13, color: COLORS.muted }); +}); + +// all-reduce arrows under the row of cards +const arrowY = gY + 154; +sketch.line(516 + 40, arrowY, 516 + 3 * (cw + gap) + cw - 40, arrowY, { + stroke: COLORS.violet.stroke, + dashed: true, +}); +sketch.text(516 + (3 * (cw + gap) + cw) / 2, arrowY + 30, '188 all-reduces per token', { + size: 18, + weight: 800, + color: COLORS.violet.stroke, +}); + +// ── footer ──────────────────────────────────────────────── +sketch.line(64, 516, W - 64, 516, { stroke: COLORS.muted, strokeWidth: 1.6 }); +sketch.text(64, 552, 'QWEN3-235B-A22B FP8 - 128 EXPERTS, 8 PER TOKEN - vLLM 0.27.1', { + size: 18, + weight: 800, + anchor: 'start', + color: COLORS.ink, +}); +sketch.text(64, 582, 'tensor, pipeline and expert parallelism explained in plain english', { + size: 16, + anchor: 'start', + color: COLORS.muted, +}); +sketch.text(W - 64, 582, 'blog.kubesimplify.com', { + size: 16, + weight: 700, + anchor: 'end', + color: COLORS.muted, +}); + +sketch.save(join(output, 'cover.svg')); +console.log(`Wrote multi-GPU vLLM cover to ${output}`); diff --git a/vercel.json b/vercel.json index 6f202d5e4..7670b27c6 100644 --- a/vercel.json +++ b/vercel.json @@ -906,6 +906,11 @@ "destination": "https://blog.kubesimplify.com/ready-for-wasm-day-2023", "permanent": true }, + { + "source": "/blog/running-a-big-llm-across-multiple-gpus-with-vllm", + "destination": "https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm", + "permanent": true + }, { "source": "/blog/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000", "destination": "https://blog.kubesimplify.com/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000", @@ -2926,6 +2931,17 @@ } ] }, + { + "source": "/running-a-big-llm-across-multiple-gpus-with-vllm", + "destination": "https://blog.kubesimplify.com/running-a-big-llm-across-multiple-gpus-with-vllm", + "permanent": true, + "has": [ + { + "type": "host", + "value": "kubesimplify.com" + } + ] + }, { "source": "/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000", "destination": "https://blog.kubesimplify.com/running-qwen3-8-flash-next-on-dgx-spark-and-rtx-pro-6000",