diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 988d9f365190..d02038cb6c5e 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -2247,6 +2247,14 @@ struct server_context_impl { void create_checkpoint(server_slot & slot, const int64_t n_tokens_cur, llama_pos pos_min, llama_pos pos_max) { const int id_task = slot.task->id; + // the buffers of an evicted checkpoint go to the new one: a fresh vector is zero-filled by resize() + std::vector spare_tgt; + std::vector spare_dft; + const auto keep_spare = [&](common_prompt_checkpoint & ckpt) { + spare_tgt = std::move(ckpt.data_tgt); + spare_dft = std::move(ckpt.data_dft); + }; + // evict checkpoints within min-step of a previous checkpoint, unless they were // created by the current task int64_t last = -1; @@ -2255,6 +2263,7 @@ struct server_context_impl { SLT_TRC(slot, "erasing context checkpoint too close to an earlier one (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", it->pos_min, it->pos_max, it->n_tokens, (float) it->size() / 1024 / 1024); + keep_spare(*it); it = slot.prompt.checkpoints.erase(it); continue; } @@ -2265,11 +2274,12 @@ struct server_context_impl { while (slot.prompt.checkpoints.size() >= (size_t) params_base.n_ctx_checkpoints) { // make room for the new checkpoint, if needed - const auto & cur = slot.prompt.checkpoints.front(); + auto & cur = slot.prompt.checkpoints.front(); SLT_WRN(slot, "erasing old context checkpoint (pos_min = %d, pos_max = %d, n_tokens = %" PRId64 ", size = %.3f MiB)\n", cur.pos_min, cur.pos_max, cur.n_tokens, (float) cur.size() / 1024 / 1024); + keep_spare(cur); slot.prompt.checkpoints.erase(slot.prompt.checkpoints.begin()); } @@ -2277,6 +2287,9 @@ struct server_context_impl { cur.id_task = id_task; + cur.data_tgt = std::move(spare_tgt); + cur.data_dft = std::move(spare_dft); + // [TAG_CHECKPOINTS_FIX_POS_MIN] // TODO: here we incorrectly deterimne that the saved checkpoint data covers the [pos_min, pos_max] range // this is not true for SWA models: https://github.com/ggml-org/llama.cpp/pull/24411#issuecomment-4677983225