From 3ce60ceb5875f287daea4c6a2bf6cd44cc5f26ad Mon Sep 17 00:00:00 2001 From: sb32445 Date: Sat, 3 Oct 2026 08:44:11 +0200 Subject: [PATCH 1/2] server : reuse the buffers of evicted prompt checkpoints create_checkpoint() evicts old checkpoints and then emplaces a new one whose data vectors are resized from empty, so every checkpoint (about 150 MiB for a hybrid model) is zero-filled and page-faulted again. Hand the buffers of the last evicted checkpoint to the next one instead. LLAMA_CKPT_REUSE=0 turns this off. On a growing multi-turn conversation this saves about 18 ms per turn (645 -> 627 ms); decode speed and outputs are unchanged. Co-Authored-By: Claude Sonnet 5.5 --- tools/server/server-context.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 988d9f365190..26e6d101fb34 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2247,6 +2247,18 @@ struct server_context_impl { void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { const int id_task = slot.task->id; + // the buffers of an evicted checkpoint go to the new one: a fresh vector is zero-filled by resize() + // (LLAMA_CKPT_REUSE=0 turns this off) + static const bool reuse = [] { const char * e = getenv("LLAMA_CKPT_REUSE"); return !e || atoi(e) != 0; }(); + std::vector spare_tgt; + std::vector spare_dft; + const auto keep_spare = [&](common_prompt_checkpoint & ckpt) { + if (reuse) { + spare_tgt = std::move(ckpt.data_tgt); + spare_dft = std::move(ckpt.data_dft); + } + }; + // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task int64_t last = -1; @@ -2255,6 +2267,7 @@ struct server_context_impl { SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024); + keep_spare(*it); it = slot.prompt.checkpoints.erase(it); continue; } @@ -2265,11 +2278,12 @@ struct server_context_impl { while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) { // make room for the new checkpoint, if needed - const auto & cur = slot.prompt.checkpoints.front(); + auto & cur = slot.prompt.checkpoints.front(); SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); + keep_spare(cur); slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin()); } @@ -2277,6 +2291,9 @@ struct server_context_impl { cur.id_task = id_task; + cur.data_tgt = std::move(spare_tgt); + cur.data_dft = std::move(spare_dft); + // [TAG_CHECKPOINTS_FIX_POS_MIN] // TODO: here we incorrectly deterimne that the saved checkpoint data covers the [pos_min, pos_max] range // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225 From ca0c09bca3ca943f8f2ae5537d96908fb124dac1 Mon Sep 17 00:00:00 2001 From: sb32445 Date: Sat, 3 Oct 2026 14:58:06 +0200 Subject: [PATCH 2/2] server : remove the LLAMA_CKPT_REUSE switch The environment switch of the previous commit was only there to measure the change; evicted checkpoint buffers are now always handed to the next checkpoint. Co-Authored-By: Claude Sonnet 5.5 --- tools/server/server-context.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 26e6d101fb34..d02038cb6c5e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2248,15 +2248,11 @@ struct server_context_impl { const int id_task = slot.task->id; // the buffers of an evicted checkpoint go to the new one: a fresh vector is zero-filled by resize() - // (LLAMA_CKPT_REUSE=0 turns this off) - static const bool reuse = [] { const char * e = getenv("LLAMA_CKPT_REUSE"); return !e || atoi(e) != 0; }(); std::vector spare_tgt; std::vector spare_dft; const auto keep_spare = [&](common_prompt_checkpoint & ckpt) { - if (reuse) { - spare_tgt = std::move(ckpt.data_tgt); - spare_dft = std::move(ckpt.data_dft); - } + spare_tgt = std::move(ckpt.data_tgt); + spare_dft = std::move(ckpt.data_dft); }; // evict checkpoints within min-step of a previous checkpoint, unless they were