From b1536f9f1bf46defa29a222b1ff9fd1002c5a714 Mon Sep 17 00:00:00 2001 From: reoring Date: Thu, 19 Nov 2026 13:51:00 +0900 Subject: [PATCH] image: make the model layer identical across releases Every release rebuilt the 2.9 GiB model layer with new file mtimes, so its digest changed although the model did not, and every version update made nodes download it again. On Craftsman that made an update miss the Knative progress deadline (APT-1342). Build the model in its own stage, normalise permissions and timestamps, and copy the tree as one layer; verify the model manifest digest so a moved upstream tag fails the build; pin BuildKit and gzip level so the layer serialisation is fixed even on a cold build. --- .github/workflows/publish.yml | 6 +++++- Dockerfile | 33 +++++++++++++++++++++++---------- README.md | 23 +++++++++++++++++++---- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a87efdc..4d004c4 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -33,6 +33,9 @@ jobs: with: persist-credentials: false - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e + with: + # Keep layer tar/gzip serialization fixed across cold release builds. + driver-opts: image=moby/buildkit:v0.22.0@sha256:832fa7aa1eb3deb56fa62ae933bfa42dad9a83ff4824dbbaf173b49c722b59d0 - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 with: registry: ghcr.io @@ -54,7 +57,8 @@ jobs: ARCH: ${{ matrix.arch }} run: | set -euo pipefail - docker buildx build --platform "linux/$ARCH" --push \ + docker buildx build --platform "linux/$ARCH" \ + --output type=image,push=true,compression=gzip,compression-level=6 \ --tag "ghcr.io/appthrust/llm:$VERSION-$ARCH" \ --label "org.opencontainers.image.source=https://github.com/appthrust/llm" \ --label "org.opencontainers.image.version=$VERSION" \ diff --git a/Dockerfile b/Dockerfile index da99a22..4f61d96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1 +# syntax=docker/dockerfile:1.18.0@sha256:dabfc0969b935b2080555ace70ee69a5261af8a8f1b4df97b9e7fbcf6722eddf FROM --platform=$BUILDPLATFORM docker.io/golang:1.27.3 AS server-builder WORKDIR /workspace @@ -13,17 +13,19 @@ RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/g go build -trimpath -ldflags='-s -w' -o /out/llm-server ./cmd/server # Stable release and multi-architecture index shared with Embedding. -FROM ollama/ollama:0.40.4@sha256:926d40fd6eeeebd346b5c358a63de2a10ec995371cd93dd22f1843200694b1a6 +FROM ollama/ollama:0.40.4@sha256:926d40fd6eeeebd346b5c358a63de2a10ec995371cd93dd22f1843200694b1a6 AS ollama-base ENV LLM_MODEL=gemma3:4b \ LLM_MAX_PROMPT_TOKENS=3000 \ OLLAMA_MODELS=/models \ OLLAMA_HOST=127.0.0.1:11434 \ HOME=/home/nonroot -# Pull at image build time; runtime never needs to download a model. Do not -# distribute the temporary Ollama identity generated by the build's daemon. + +FROM ollama-base AS model-builder +# Export only the model and runtime home, never daemon identity or temporary +# directory metadata. Verify the manifest so a moved upstream tag fails closed. RUN set -eu; \ - export HOME=/tmp/ollama-build; \ - ollama serve & server=$!; \ + export HOME=/tmp/ollama-build OLLAMA_MODELS=/image-root/models; \ + ollama serve >/tmp/ollama-build.log 2>&1 & server=$!; \ trap 'kill "$server" 2>/dev/null || true; wait "$server" 2>/dev/null || true' EXIT; \ attempts=0; \ until ollama list >/dev/null 2>&1; do \ @@ -33,13 +35,24 @@ RUN set -eu; \ sleep 1; \ done; \ ollama pull "$LLM_MODEL"; \ + printf '%s %s\n' \ + '68ceab7e468a31c988b44d5a4984a0e21f58c4f7158c6ce29c1804babac38df7' \ + "$OLLAMA_MODELS/manifests-v2/ollama.com/library/gemma3/4b" | sha256sum -c -; \ kill "$server"; \ wait "$server" || true; \ trap - EXIT; \ - rm -rf /tmp/ollama-build; \ - mkdir -p /home/nonroot; \ - chown -R 65532:65532 /home/nonroot; \ - chmod -R a+rX,a-w /models + rm -rf /tmp/ollama-build /tmp/ollama-build.log; \ + mkdir -p /image-root/home/nonroot; \ + chown 65532:65532 /image-root/home/nonroot; \ + find /image-root/models -type d -exec chmod 0555 {} +; \ + find /image-root/models -type f -exec chmod 0444 {} +; \ + chmod 0755 /image-root /image-root/home /image-root/home/nonroot; \ + find /image-root -exec touch -h -d '@0' {} + + +FROM ollama-base +# A sorted COPY of a fixed-content, fixed-metadata tree is independent of the +# pull time and server revision, even with a cold build cache. +COPY --link --from=model-builder /image-root/ / COPY --from=server-builder /out/llm-server /llm-server USER 65532 EXPOSE 8080 diff --git a/README.md b/README.md index 88bdcfa..fa9cc73 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,8 @@ For a local server with an existing Ollama instance and model: LLM_MODEL=gemma3:4b LLM_OLLAMA_BIN= go run ./cmd/server --listen :8080 ``` -Build the image on its native architecture (the final stage runs Ollama to bake -the model). A full build downloads the model; a builder-only check does not: +Build the image on its native architecture (the `model-builder` stage runs +Ollama). A full build downloads the model; a builder-only check does not: ```sh docker build --platform linux/amd64 -t llm-local . @@ -46,8 +46,23 @@ docker build --platform linux/amd64 --target server-builder -t llm-builder . The image preserves the pinned Ollama `0.40.4@sha256:926d40fd6eeeebd346b5c358a63de2a10ec995371cd93dd22f1843200694b1a6` -base shared with Embedding. Temporary build-time Ollama identity files are -removed. The image exposes port 8080 and starts `/llm-server` as UID 65532. +base shared with Embedding. The model stage verifies the `gemma3:4b` manifest +against SHA-256 `68ceab7e468a31c988b44d5a4984a0e21f58c4f7158c6ce29c1804babac38df7`; +an upstream tag change fails the build instead of silently changing the model. +An intentional model update must update this checksum and, when switching +models, the model name and manifest path together. +Only the model tree and an empty, UID-65532-owned runtime home enter the final +image. Build-daemon identity, logs, and temporary-directory metadata do not. +Model files/directories have fixed read-only modes and all exported timestamps +are Unix epoch zero. The separate model `COPY --link` precedes the server copy. + +Publication pins the Dockerfile frontend and BuildKit and explicitly uses gzip +level 6, so rebuilding without a model-step cache produces the same compressed +model layer. Release labels and server changes do not affect that layer. The +first release using this layout downloads a new model layer once; later +server-only releases reuse it. Changing the model, layout, or serialization +toolchain can intentionally change the layer digest and needs a fresh cold-build +comparison. The image exposes port 8080 and starts `/llm-server` as UID 65532. Only `go.mod`, `go.sum`, and `cmd/server` enter the Go builder. ## HTTP contract