Release: stable prompt-cache prefix and keys, token usage metrics

Dockerfile

@@ -5,13 +5,13 @@
 # ================================
 # Multi-stage build for Frontend and Backend
 # Switched to Debian/Ubuntu-based images to resolve Alpine network issues
-# Node is sourced from public.ecr.aws; Liberica Java images come from BellSoft's Docker Hub repositories
+# Base images pull through the fleet's dockerhub.haiku.host mirror; anonymous public.ecr.aws pulls hit 429 data limits in CI.
 # Note: Requires DOCKER_BUILDKIT=1 for cache mount support
 
 # ================================
 # FRONTEND BUILD STAGE
 # ================================
-FROM public.ecr.aws/docker/library/node:24.18.0-bookworm-slim AS frontend-builder
+FROM dockerhub.haiku.host/library/node:24.18.0-bookworm-slim AS frontend-builder
 WORKDIR /app/frontend
 
 # Copy dependency definitions first for cache layer
@@ -33,13 +33,13 @@
 # which is why this is a plain ARG rather than a BuildKit secret. Declared last
 # so a key change invalidates only the build layer below it. Left unset, the
 # frontend starts with authentication disabled (clerkAuthentication.svelte.ts).
 ARG VITE_CLERK_PUBLISHABLE_KEY
 RUN npm run validate && npm run test && npm run build
 
 # ================================
 # BACKEND BUILD STAGE
 # ================================
-FROM bellsoft/liberica-openjdk-debian:25.0.3-11 AS builder
+FROM dockerhub.haiku.host/bellsoft/liberica-openjdk-debian:25.0.3-11 AS builder
 ARG SOURCE_COMMIT=unknown
 WORKDIR /app
 
@@ -124,7 +124,7 @@
 # ================================
 # RUNTIME STAGE
 # ================================
-FROM bellsoft/liberica-openjre-debian:25.0.3-11 AS runtime
+FROM dockerhub.haiku.host/bellsoft/liberica-openjre-debian:25.0.3-11 AS runtime
 LABEL io.iocloudhost.logs.owner=split
 
 # 1. System packages (never changes) - FIRST for maximum cache reuse

frontend/scripts/verify-node-version.mjs

@@ -24,7 +24,7 @@ const gettingStartedGuide = await readFile(
 
 const expectedNodeVersionFile = `v${canonicalNodeVersion}`;
 const projectedLockfileVersion = frontendLockfile.packages?.[""]?.engines?.node;
-const expectedDockerImage = `public.ecr.aws/docker/library/node:${canonicalNodeVersion}-bookworm-slim`;
+const expectedDockerImage = `dockerhub.haiku.host/library/node:${canonicalNodeVersion}-bookworm-slim`;
 const expectedDocumentationVersion = `Node.js ${canonicalNodeVersion}`;
 
 if (nodeVersionFile !== expectedNodeVersionFile) {

src/main/java/com/williamcallahan/javachat/application/prompt/PromptTruncator.java

@@ -45,8 +45,9 @@ public class PromptTruncator {
      * @throws AuthoritativeContextDoesNotFitException when no HIGH-priority context segment fits
      */
     public TruncatedPrompt truncate(StructuredPrompt prompt, int maxTokens) {
-        int reservedTokens =
-                prompt.system().estimatedTokens() + prompt.currentQuery().estimatedTokens();
+        int reservedTokens = prompt.system().estimatedTokens()
+                + prompt.currentQuery().estimatedTokens()
+                + prompt.turnGuidance().estimatedTokens();
         List<ContextDocumentSegment> authoritativeContextDocuments = prompt.contextDocuments().stream()
                 .filter(contextDocument -> contextDocument.priority() == PromptSegmentPriority.HIGH)
                 .toList();
@@ -58,8 +59,8 @@ public TruncatedPrompt truncate(StructuredPrompt prompt, int maxTokens) {
                     prompt.currentQuery().estimatedTokens(),
                     maxTokens);
             // Return prompt with only system and query - no room for context or history
-            StructuredPrompt minimalPrompt =
-                    new StructuredPrompt(prompt.system(), List.of(), List.of(), prompt.currentQuery());
+            StructuredPrompt minimalPrompt = new StructuredPrompt(
+                    prompt.system(), List.of(), List.of(), prompt.currentQuery(), prompt.turnGuidance());
             return new TruncatedPrompt(minimalPrompt, true);
         }
 
@@ -96,8 +97,8 @@ public TruncatedPrompt truncate(StructuredPrompt prompt, int maxTokens) {
             log.debug("Truncated context documents from {} to {}", originalDocCount, fittingDocs.size());
         }
 
-        StructuredPrompt truncated =
-                new StructuredPrompt(prompt.system(), reindexedDocuments, fittingTurns, prompt.currentQuery());
+        StructuredPrompt truncated = new StructuredPrompt(
+                prompt.system(), reindexedDocuments, fittingTurns, prompt.currentQuery(), prompt.turnGuidance());
 
         if (wasTruncated) {
             log.info(

src/main/java/com/williamcallahan/javachat/domain/prompt/CompletionPrompt.java

@@ -0,0 +1,12 @@
+package com.williamcallahan.javachat.domain.prompt;
+
+import java.util.Objects;
+
+/** Separates reusable task instructions from the current completion's untrusted input. */
+public record CompletionPrompt(String instructions, String input) {
+    /** Requires both boundary fields so instruction omission is explicit. */
+    public CompletionPrompt {
+        Objects.requireNonNull(instructions, "instructions");
+        Objects.requireNonNull(input, "input");
+    }
+}

src/main/java/com/williamcallahan/javachat/domain/prompt/StructuredPrompt.java

@@ -14,16 +14,32 @@
  * @param contextDocuments authoritative HIGH or ordinary LOW context documents
  * @param conversationHistory past conversation turns (MEDIUM priority)
  * @param currentQuery the user's current question (HIGH priority)
+ * @param turnGuidance application-owned guidance specific to the current turn
  */
 public record StructuredPrompt(
         SystemSegment system,
         List<ContextDocumentSegment> contextDocuments,
         List<ConversationTurnSegment> conversationHistory,
-        CurrentQuerySegment currentQuery) {
+        CurrentQuerySegment currentQuery,
+        SystemSegment turnGuidance) {
 
     /** Separator between prompt segments. */
     public static final String SEGMENT_SEPARATOR = "\n\n";
 
+    /** Creates a prompt without turn-specific guidance. */
+    public StructuredPrompt(
+            SystemSegment system,
+            List<ContextDocumentSegment> contextDocuments,
+            List<ConversationTurnSegment> conversationHistory,
+            CurrentQuerySegment currentQuery) {
+        this(system, contextDocuments, conversationHistory, currentQuery, new SystemSegment("", 0));
+    }
+
+    /** Keeps turn-specific instructions outside the reusable system prefix. */
+    public StructuredPrompt withTurnGuidance(SystemSegment guidance) {
+        return new StructuredPrompt(system, contextDocuments, conversationHistory, currentQuery, guidance);
+    }
+
     /**
      * Creates a structured prompt with defensive copies of lists.
      *
@@ -36,6 +52,7 @@ public record StructuredPrompt(
         if (currentQuery == null) {
             throw new IllegalArgumentException("Current query segment cannot be null");
         }
+        java.util.Objects.requireNonNull(turnGuidance, "turnGuidance");
         contextDocuments = contextDocuments == null ? List.of() : List.copyOf(contextDocuments);
         conversationHistory = conversationHistory == null ? List.of() : List.copyOf(conversationHistory);
     }
@@ -46,7 +63,7 @@ public record StructuredPrompt(
      * @return sum of all segment token estimates
      */
     public int totalEstimatedTokens() {
-        int total = system.estimatedTokens() + currentQuery.estimatedTokens();
+        int total = system.estimatedTokens() + currentQuery.estimatedTokens() + turnGuidance.estimatedTokens();
         for (ContextDocumentSegment contextDocument : contextDocuments) {
             total += contextDocument.estimatedTokens();
         }
@@ -60,7 +77,7 @@ public int totalEstimatedTokens() {
      * Renders the complete prompt to a string for consumers that require one text value.
      *
      * <p>Segments are joined with paragraph separators in order: system instructions,
-     * context documents, conversation history, and the current query.</p>
+     * conversation history, turn guidance, context documents, and the current query.</p>
      *
      * @return the fully assembled prompt string
      */
@@ -79,13 +96,15 @@ public String render() {
     public String renderInput() {
         StringJoiner promptInput = new StringJoiner(SEGMENT_SEPARATOR);
 
-        for (ContextDocumentSegment contextDocument : contextDocuments) {
-            promptInput.add(contextDocument.content());
-        }
-
         for (ConversationTurnSegment conversationTurn : conversationHistory) {
             promptInput.add(conversationTurn.content());
         }
+        if (!turnGuidance.content().isBlank()) {
+            promptInput.add(turnGuidance.content());
+        }
+        for (ContextDocumentSegment contextDocument : contextDocuments) {
+            promptInput.add(contextDocument.content());
+        }
 
         promptInput.add(currentQuery.content());
 
@@ -99,7 +118,7 @@ public String renderInput() {
      * @return new prompt with updated documents
      */
     public StructuredPrompt withContextDocuments(List<ContextDocumentSegment> newContextDocuments) {
-        return new StructuredPrompt(system, newContextDocuments, conversationHistory, currentQuery);
+        return new StructuredPrompt(system, newContextDocuments, conversationHistory, currentQuery, turnGuidance);
     }
 
     /**
@@ -109,7 +128,7 @@ public StructuredPrompt withContextDocuments(List<ContextDocumentSegment> newCon
      * @return new prompt with updated history
      */
     public StructuredPrompt withConversationHistory(List<ConversationTurnSegment> newConversationHistory) {
-        return new StructuredPrompt(system, contextDocuments, newConversationHistory, currentQuery);
+        return new StructuredPrompt(system, contextDocuments, newConversationHistory, currentQuery, turnGuidance);
     }
 
     /**

src/main/java/com/williamcallahan/javachat/service/ChatService.java

@@ -118,13 +118,9 @@ public RetrievalService.CitationOutcome citationOutcomeForRetainedContext(
     public StructuredPrompt buildStructuredPromptWithContextAndGuidance(
             List<Message> history, String latestUserMessage, List<Document> contextDocs, String guidance) {
 
-        // Build system prompt with guidance
         String basePrompt = systemPromptConfig.getCoreSystemPrompt();
-        String completePrompt = guidance != null && !guidance.isBlank()
-                ? systemPromptConfig.buildFullPrompt(basePrompt, guidance)
-                : basePrompt;
-
-        SystemSegment systemSegment = new SystemSegment(completePrompt, estimateTokens(completePrompt));
+        SystemSegment systemSegment = new SystemSegment(basePrompt, estimateTokens(basePrompt));
+        String turnGuidance = guidance == null ? "" : guidance;
 
         List<ContextDocumentSegment> contextSegments =
                 buildContextSegments(contextDocs != null ? contextDocs : List.of(), latestUserMessage);
@@ -134,7 +130,8 @@ public StructuredPrompt buildStructuredPromptWithContextAndGuidance(
         CurrentQuerySegment querySegment =
                 new CurrentQuerySegment(latestUserMessage, estimateTokens(latestUserMessage));
 
-        return new StructuredPrompt(systemSegment, contextSegments, conversationSegments, querySegment);
+        return new StructuredPrompt(systemSegment, contextSegments, conversationSegments, querySegment)
+                .withTurnGuidance(new SystemSegment(turnGuidance, estimateTokens(turnGuidance)));
     }
 
     /**
@@ -195,27 +192,26 @@ public StructuredPromptOutcome buildStructuredPromptWithContextOutcome(
 
         String searchQualityNote = determineSearchQuality(contextDocs);
 
-        // Build system prompt with search quality context
-        StringBuilder systemPromptBuilder = new StringBuilder(systemPromptConfig.getCoreSystemPrompt());
+        String coreSystemPrompt = systemPromptConfig.getCoreSystemPrompt();
+        StringBuilder searchGuidance = new StringBuilder();
         if (!searchQualityNote.isEmpty()) {
-            systemPromptBuilder.append("\n\nSEARCH CONTEXT: ").append(searchQualityNote);
+            searchGuidance.append("SEARCH CONTEXT: ").append(searchQualityNote);
             if (searchQualityNote.contains("less relevant") || searchQualityNote.contains("keyword search")) {
-                systemPromptBuilder.append("\n").append(systemPromptConfig.getLowQualitySearchPrompt());
+                searchGuidance.append("\n").append(systemPromptConfig.getLowQualitySearchPrompt());
             }
         }
-        String systemPromptText = systemPromptBuilder.toString();
-
-        // Build structured segments
-        SystemSegment systemSegment = new SystemSegment(systemPromptText, estimateTokens(systemPromptText));
+        SystemSegment systemSegment = new SystemSegment(coreSystemPrompt, estimateTokens(coreSystemPrompt));
+        String turnGuidance = searchGuidance.toString();
 
         List<ContextDocumentSegment> contextSegments = buildContextSegments(contextDocs, latestUserMessage);
         List<ConversationTurnSegment> conversationSegments = buildConversationSegments(history);
 
         CurrentQuerySegment querySegment =
                 new CurrentQuerySegment(latestUserMessage, estimateTokens(latestUserMessage));
 
-        StructuredPrompt structuredPrompt =
-                new StructuredPrompt(systemSegment, contextSegments, conversationSegments, querySegment);
+        StructuredPrompt structuredPrompt = new StructuredPrompt(
+                        systemSegment, contextSegments, conversationSegments, querySegment)
+                .withTurnGuidance(new SystemSegment(turnGuidance, estimateTokens(turnGuidance)));
 
         return new StructuredPromptOutcome(structuredPrompt, contextDocs);
     }

src/main/java/com/williamcallahan/javachat/service/EmbeddingBatchEmbedder.java

@@ -23,6 +23,7 @@ final class EmbeddingBatchEmbedder {
     static final int MAX_CONCURRENT_EMBEDDING_REQUESTS = 8;
     static final int MAX_EMBEDDING_ATTEMPTS = 3;
     private static final Duration EMBEDDING_RETRY_BASE_DELAY = Duration.ofSeconds(5);
+    private static final Duration EMBEDDING_BATCH_RETRY_TIMEOUT = Duration.ofMinutes(10);
 
     private EmbeddingBatchEmbedder() {}
 
@@ -158,23 +159,42 @@ private static List<float[]> embedSingleBatch(
 
     private static List<float[]> embedWithRetry(
             LlmGatewayTier.Purpose purpose, EmbeddingClient embeddingClient, List<String> textBatch) {
-        for (int embeddingAttempt = 1; embeddingAttempt <= MAX_EMBEDDING_ATTEMPTS; embeddingAttempt++) {
+        long retryDeadlineNanos = System.nanoTime() + EMBEDDING_BATCH_RETRY_TIMEOUT.toNanos();
+        int providerAttemptCount = 0;
+        while (providerAttemptCount < MAX_EMBEDDING_ATTEMPTS) {
+            Duration remainingBudget = Duration.ofNanos(Math.max(0, retryDeadlineNanos - System.nanoTime()));
+            if (remainingBudget.isZero()) {
+                throw new EmbeddingServiceTemporarilyUnavailableException("Embedding batch retry deadline elapsed");
+            }
             try {
-                return embeddingClient.embed(purpose, textBatch, LlmGatewayTier.BATCH);
+                return embeddingClient.embed(purpose, textBatch, LlmGatewayTier.BATCH, remainingBudget);
             } catch (EmbeddingServiceTemporarilyUnavailableException temporaryFailure) {
-                if (embeddingAttempt == MAX_EMBEDDING_ATTEMPTS) {
+                if (temporaryFailure.providerContacted()) {
+                    providerAttemptCount++;
+                }
+                if (providerAttemptCount == MAX_EMBEDDING_ATTEMPTS) {
+                    throw temporaryFailure;
+                }
+                Duration retryDelay = temporaryFailure.retryDelay();
+                Duration backoff = temporaryFailure.providerContacted()
+                        ? EMBEDDING_RETRY_BASE_DELAY.multipliedBy(providerAttemptCount)
+                        : Duration.ZERO;
+                if (backoff.compareTo(retryDelay) > 0) {
+                    retryDelay = backoff;
+                }
+                if (retryDelay.toNanos() >= retryDeadlineNanos - System.nanoTime()) {
                     throw temporaryFailure;
                 }
-                awaitRetryDelay(embeddingAttempt, temporaryFailure);
+                awaitRetryDelay(retryDelay, temporaryFailure);
             }
         }
         throw new IllegalStateException("Embedding retry loop exhausted without a terminal outcome");
     }
 
     private static void awaitRetryDelay(
-            int completedAttemptCount, EmbeddingServiceTemporarilyUnavailableException temporaryFailure) {
+            Duration retryDelay, EmbeddingServiceTemporarilyUnavailableException temporaryFailure) {
         try {
-            Thread.sleep(EMBEDDING_RETRY_BASE_DELAY.multipliedBy(completedAttemptCount));
+            Thread.sleep(retryDelay);
         } catch (InterruptedException interruptedException) {
             Thread.currentThread().interrupt();
             EmbeddingServiceTemporarilyUnavailableException interruptionFailure =

src/main/java/com/williamcallahan/javachat/service/EmbeddingServiceTemporarilyUnavailableException.java

@@ -1,20 +1,23 @@
 package com.williamcallahan.javachat.service;
 
+import java.time.Duration;
+
 /**
  * Signals that embedding work can be retried after a transient admission or provider failure.
  *
  * <p>Non-final so the embedding client can distinguish a locally recorded provider cooldown
  * (no provider contact made) from a genuine transient provider failure through subtyping.</p>
  */
 public class EmbeddingServiceTemporarilyUnavailableException extends EmbeddingServiceUnavailableException {
+    private final long retryNotBeforeNanos;
 
     /**
      * Creates a retryable embedding failure with a human-readable explanation.
      *
      * @param message explanation of the transient failure
      */
     public EmbeddingServiceTemporarilyUnavailableException(String message) {
-        super(message);
+        this(message, null, Duration.ZERO);
     }
 
     /**
@@ -24,6 +27,22 @@ public EmbeddingServiceTemporarilyUnavailableException(String message) {
      * @param cause underlying provider or transport failure
      */
     public EmbeddingServiceTemporarilyUnavailableException(String message, Throwable cause) {
+        this(message, cause, Duration.ZERO);
+    }
+
+    /** Preserves authoritative provider retry timing across the batch boundary. */
+    public EmbeddingServiceTemporarilyUnavailableException(String message, Throwable cause, Duration retryDelay) {
         super(message, cause);
+        retryNotBeforeNanos = System.nanoTime() + retryDelay.toNanos();
+    }
+
+    /** Returns the remaining authoritative cooldown without parsing diagnostic text. */
+    public Duration retryDelay() {
+        return Duration.ofNanos(Math.max(0, retryNotBeforeNanos - System.nanoTime()));
+    }
+
+    /** Distinguishes provider failures from local admission rejection for retry budgeting. */
+    public boolean providerContacted() {
+        return true;
     }
 }