feat: bring /v1/decision onto perf

common/arg.cpp

@@ -2555,6 +2555,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
                 params.n_parallel = value;
             }
         ).set_env("LLAMA_ARG_N_PARALLEL").set_examples({LLAMA_EXAMPLE_SERVER}));
+        add_opt(common_arg(
+            {"--decision-seqs"}, "N",
+            string_format("sequences reserved for the /decision endpoint, above the slots; enables it (default: %d = disabled, minimum 3)", params.n_seq_decision),
+            [](common_params & params, int value) {
+                if (value != 0 && value < 3) {
+                    throw std::invalid_argument("--decision-seqs needs at least 3 (cached prefix, trunk, one branch)");
+                }
+                params.n_seq_decision = value;
+            }
+        ).set_env("LLAMA_ARG_DECISION_SEQS").set_examples({LLAMA_EXAMPLE_SERVER}));
     } else {
         add_opt(common_arg(
             {"-np", "--parallel"}, "N",

common/common.cpp

@@ -1725,7 +1725,7 @@ struct llama_context_params common_context_params_to_llama(const common_params &
     auto cparams = llama_context_default_params();
 
     cparams.n_ctx             = params.n_ctx;
-    cparams.n_seq_max         = params.n_parallel;
+    cparams.n_seq_max         = params.n_parallel + params.n_seq_decision; // decision sequences sit above the slots
     cparams.n_rs_seq          = params.speculative.need_n_rs_seq();
     cparams.n_outputs_max     = std::max(params.n_outputs_max, 0);
     cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0);

common/common.h

@@ -453,6 +453,7 @@ struct common_params {
     int32_t n_keep                =     0; // number of tokens to keep from initial prompt
     int32_t n_chunks              =    -1; // max number of chunks to process (-1 = unlimited)
     int32_t n_parallel            =     1; // number of parallel sequences to decode
+    int32_t n_seq_decision        =     0; // sequences reserved for llama-server's /decision endpoint (0 = disabled)
     int32_t n_sequences           =     1; // number of sequences to decode
     int32_t n_outputs_max         =     0; // max outputs in a batch (0 = n_batch)
     int32_t n_outputs_max_per_seq =     1; // max outputs per sequence

common/speculative.cpp

@@ -2480,6 +2480,7 @@ common_params common_base_params_to_speculative(const common_params & params) {
 
     result.cache_type_k  = params_spec.cache_type_k;
     result.cache_type_v  = params_spec.cache_type_v;
+    result.n_seq_decision = 0; // decision sequences live on the main context only
     result.n_outputs_max = params.n_parallel;
     result.n_outputs_max_per_seq = 1;
 

tools/CMakeLists.txt

@@ -19,6 +19,7 @@ else()
     add_subdirectory(imatrix)
     add_subdirectory(moe-trace)
     add_subdirectory(llama-bench)
+    add_subdirectory(parallel-decision)
     add_subdirectory(completion)
     add_subdirectory(perplexity)
     add_subdirectory(quantize)

tools/parallel-decision/CMakeLists.txt

@@ -0,0 +1,15 @@
+# shared engine: used by llama-parallel-decision and by llama-server's /decision endpoint
+add_library(llama-decision STATIC decision-engine.cpp decision-engine.h)
+target_include_directories(llama-decision PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+target_link_libraries(llama-decision PUBLIC llama-common llama)
+target_compile_features(llama-decision PUBLIC cxx_std_17)
+# llama-server links it into a shared library (libllama-server-impl)
+set_target_properties(llama-decision PROPERTIES POSITION_INDEPENDENT_CODE ON)
+
+set(TARGET llama-parallel-decision)
+add_executable(${TARGET} parallel-decision.cpp)
+if(LLAMA_TOOLS_INSTALL)
+    install(TARGETS ${TARGET} RUNTIME)
+endif()
+target_link_libraries(${TARGET} PRIVATE llama-decision llama-common llama ${CMAKE_THREAD_LIBS_INIT})
+target_compile_features(${TARGET} PRIVATE cxx_std_17)

tools/parallel-decision/README.md

@@ -0,0 +1,134 @@
+# parallel-decision
+
+Answer a finite JSON schema in one batched forward pass, instead of generating the JSON token by token.
+
+Every field of a schema has a fixed set of allowed values (enums, booleans, bounded integers, numbers on a grid).
+After the context, each field's allowed values are scored as token paths that fork from the same KV cache, so all
+fields are answered in one `llama_decode` and cannot see each other. Each answer comes back with a probability, and
+the JSON object is assembled by code, so it always matches the schema.
+
+This directory holds the engine (`decision-engine.*`), a CLI (`llama-parallel-decision`), and the engine is also
+served by `llama-server` as `POST /v1/decision`.
+
+## Build
+
+Same as llama.cpp:
+
+```bash
+cmake -B build -DGGML_CUDA=ON        # or plain `cmake -B build` for CPU / Metal
+cmake --build build --config Release -j
+```
+
+## Run the server
+
+`--decision-seqs N` reserves the sequence slots the decisions need: one holds the cached instructions, one per context
+in flight, the rest are the parallel questions. It also switches the KV cache to unified, which is what lets the
+branches share the context's cells.
+
+```bash
+./build/bin/llama-server -m model.gguf -ngl 99 -fa on -c 32768 --decision-seqs 24 --port 8096
+```
+
+With a presets file, one loaded model serves chat and decisions:
+
+```ini
+[*]
+ngl = 99
+fa = on
+jinja = 1
+parallel = 1
+cache-type-k = q8_0
+cache-type-v = q8_0
+decision-seqs = 24
+
+[gemma-4-12b]
+model = ./models/gemma-4-12b-it-UD-Q4_K_XL.gguf
+ctx-size = 32768
+ubatch-size = 512
+decision-seqs = 12
+```
+
+```bash
+./build/bin/llama-server --models-preset models.ini --models-max 1 --port 8096
+```
+
+How many sequences a model affords depends on its attention. A plain-attention model shares the context's cells, so
+128 sequences cost almost nothing. A sliding-window model (Gemma) allocates its window per sequence, so keep it low
+(12 on a 12 GB card). Hybrid models with recurrent layers (Qwen3.5, Nemotron-H) keep a recurrent state per
+sequence (about 50 MB each for Qwen3.5 4B and 9B), and llama.cpp only batches their sequences together when they hold
+the same number of tokens. The engine right-pads each group of branches to its longest one, so they still score in a
+single pass; the padding comes after the token that is read, so it doesn't change the result.
+
+## POST /v1/decision
+
+`contexts` is a list of 1-256 strings. They share one schema, one set of instructions, and one cached prefix; results
+come back in the same order.
+
+```bash
+curl http://localhost:8096/v1/decision -H "Content-Type: application/json" -d '{
+  "model": "gemma-4-12b",
+  "instructions": "Answer each question about this support request from its state.",
+  "schema": {
+    "category": {"type": "enum", "choices": ["billing","technical","cancellation","other"],
+                 "description": "What type of support request is this?"},
+    "urgent":   {"type": "boolean", "description": "Does this need urgent handling?"},
+    "priority": {"type": "enum", "choices": ["low","medium","high","critical"],
+                 "description": "Rate support priority."}
+  },
+  "contexts": ["I was charged twice and need this fixed today."]
+}'
+```
+
+```json
+{
+  "object": "decision",
+  "results": [
+    {
+      "decision": {"category": "billing", "urgent": true, "priority": "high"},
+      "fields": {
+        "category": {"value": "billing",  "probability": 1.0,  "scored_nodes": 1, "tree": true},
+        "urgent":   {"value": true,       "probability": 1.0,  "scored_nodes": 1, "tree": true},
+        "priority": {"value": "high",     "probability": 0.74, "scored_nodes": 1, "tree": true}
+      },
+      "usage": {"context_tokens": 21, "scored_rows": 14}
+    }
+  ],
+  "usage": {"prompt_tokens": 137, "cached_tokens": 116, "context_tokens": 21, "scored_rows": 14},
+  "timings": {"prefill_ms": 50.7, "scoring_ms": 50.0, "total_ms": 100.7, "rounds": 1, "per_decision_ms": 100.7}
+}
+```
+
+(That response is a real one: Gemma 4 12B on an RTX 3060, warm cache.)
+
+### Schema
+
+Compact fields, or a JSON Schema object with `properties`:
+
+| type | keys | notes |
+|---|---|---|
+| `enum` | `choices` (or `enum`) | 1-255 values |
+| `boolean` | - | true / false |
+| `integer` | `minimum`, `maximum` | 1-255 values |
+| `number` | `minimum`, `maximum`, `step` (`multipleOf` in JSON Schema) | fixed-width decimals |
+
+Numeric fields take `aggregate`: `mode` (default), `median` or `mean`.
+
+### Options
+
+| field | default | meaning |
+|---|---|---|
+| `instructions` | `""` | prepended to the generated field catalogue; cached with it |
+| `mode` | `auto` | `tree` scores every divergence node and returns exact probabilities; `greedy` walks the trie; `auto` picks tree up to `tree_max` values |
+| `tree_max` | 128 | per-field switch between tree and greedy |
+| `cache_prompt` | true | reuse the cached instructions + schema prefix |
+
+## CLI
+
+`llama-parallel-decision` runs the same engine from a worker process (stdin/stdout protocol, one JSON request per
+line). Environment: `DECIDE_TREE`, `DECIDE_TREE_MAX`, `DECIDE_NSEQ`, `DECIDE_SPLIT_BOUNDARY`.
+
+## A UI for it
+
+[decision-playground](https://github.com/thecodacus/decision-playground) is a browser-only playground: it talks
+straight to your llama-server, runs a decision and the same question as a chat completion side by side with live
+timers, and has a small game whose agents decide through the endpoint.