diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000000..3814594766
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,2 @@
+/start-server.sh text eol=lf
+/start-server.bat text eol=crlf
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 06485ca16c..35418c280a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -26,6 +26,8 @@ on:
         "ggml",
         "examples/server/frontend",
         "examples/server/frontend/**",
+        "start-server.sh",
+        "start-server.bat",
       ]
   pull_request:
     types: [opened, synchronize, reopened]
@@ -44,6 +46,8 @@ on:
         "ggml",
         "examples/server/frontend",
         "examples/server/frontend/**",
+        "start-server.sh",
+        "start-server.bat",
       ]
 
 env:
@@ -107,7 +111,8 @@ jobs:
         run: |
           cp ggml/LICENSE ./build/bin/ggml.txt
           cp LICENSE ./build/bin/stable-diffusion.cpp.txt
-          zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/*
+          chmod +x start-server.sh
+          zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/* ./start-server.sh ./start-server.bat
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -171,7 +176,8 @@ jobs:
         run: |
           cp ggml/LICENSE ./build/bin/ggml.txt
           cp LICENSE ./build/bin/stable-diffusion.cpp.txt
-          zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip ./build/bin/*
+          chmod +x start-server.sh
+          zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-vulkan.zip ./build/bin/* ./start-server.sh ./start-server.bat
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -326,7 +332,8 @@ jobs:
         run: |
           cp ggml/LICENSE ./build/bin/ggml.txt
           cp LICENSE ./build/bin/stable-diffusion.cpp.txt
-          zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/*
+          chmod +x start-server.sh
+          zip -j sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-${{ steps.system-info.outputs.OS_NAME }}-${{ steps.system-info.outputs.OS_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}.zip ./build/bin/* ./start-server.sh ./start-server.bat
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -418,7 +425,7 @@ jobs:
             ls .\build\bin
             throw "Can't find stable-diffusion.dll"
           }
-          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip $filePath
+          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-${{ matrix.build }}-x64.zip $filePath .\start-server.sh .\start-server.bat
 
       - name: Copy and pack Cuda runtime
         id: pack_cuda_runtime
@@ -450,7 +457,7 @@ jobs:
     runs-on: windows-2022
 
     env:
-      ROCM_VERSION: "7.14.0"
+      ROCM_VERSION: "10.1.0"
       GPU_TARGETS: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201"
       # rocm-sdk re-expands clang from the wheel every run, so its mtime always
       # differs and the default compiler_check=mtime misses every object.
@@ -502,7 +509,7 @@ jobs:
 
           write-host "Installing ROCm wheels for multi-arch support"
           # Install ROCm wheels for multi-arch support (this may take several minutes)
-          python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{env.ROCM_VERSION}}"
+          python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{env.ROCM_VERSION}}"
 
           # Pre-expand the devel tree so it is included in the cache
           write-host "Initializing ROCm devel tree"
@@ -564,10 +571,22 @@ jobs:
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
         uses: pr-mpt/actions-commit-hash@01d19a83c242e1851c9aa6cf9625092ecd095d09  # v2
 
+      - name: Bundle HIP runtime libraries
+        if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
+        run: |
+          $ErrorActionPreference = "Stop"
+          $binPath = (rocm-sdk path --bin).Trim()
+          if (-not $binPath) { throw "rocm-sdk path --bin returned empty" }
+          foreach ($dll in @("amdhip64_7.dll", "rocm_kpack.dll", "amd_comgr.dll")) {
+            $f = Get-ChildItem -Path $binPath -Filter $dll -ErrorAction SilentlyContinue
+            if (-not $f) { throw "no match for $dll in $binPath" }
+            Copy-Item $f.FullName -Destination .\build\bin\ -Force
+          }
+
       - name: Pack artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
         run: |
-          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\*
+          7z a sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-win-rocm-${{ env.ROCM_VERSION }}-x64.zip .\build\bin\* .\start-server.sh .\start-server.bat
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
@@ -587,9 +606,9 @@ jobs:
     strategy:
       matrix:
         include:
-          - ROCM_VERSION: "7.14.0"
+          - ROCM_VERSION: "10.1.0"
             gpu_targets: "gfx900;gfx906;gfx908;gfx90a;gfx90c;gfx942;gfx950;gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201"
-            build: x64
+            build: 'x64'
 
     steps:
       - name: Clone
@@ -606,6 +625,15 @@ jobs:
           # Plateaus at the 500M default (88.19% full, 142 misses/run).
           max-size: 2G
 
+      - name: Tune ccache for reinstalled ROCm toolchain
+        run: |
+          # ROCm is pip-installed fresh each run, so the clang binary's mtime
+          # changes every time. With the default compiler_check=mtime that
+          # invalidates the cache; hash compiler contents instead so warm
+          # builds hit.
+          ccache --set-config=compiler_check=content
+          ccache --set-config=sloppiness=time_macros,include_file_mtime,include_file_ctime
+
       - name: Dependencies
         id: depends
         run: |
@@ -641,7 +669,7 @@ jobs:
           # libraries = HIP runtime and CMake configs needed for linking
           # devel = compilers, headers, static libs
           python -m pip install --upgrade pip
-          python -m pip install --index-url https://repo.amd.com/rocm/whl-multi-arch/ "rocm[libraries,devel]==${{matrix.ROCM_VERSION}}"
+          python -m pip install --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries,devel]==${{matrix.ROCM_VERSION}}"
 
           # Get ROCm installation paths using the rocm-sdk CLI tool
           ROCM_PATH=$(rocm-sdk path --root)
@@ -722,7 +750,8 @@ jobs:
         run: |
           cp ggml/LICENSE ./build/bin/ggml.txt
           cp LICENSE ./build/bin/stable-diffusion.cpp.txt
-          zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip ./build/bin
+          chmod +x start-server.sh
+          zip -y -r sd-${{ env.BRANCH_NAME }}-${{ steps.commit.outputs.short }}-bin-${{ steps.system-info.outputs.OS_TYPE }}-Ubuntu-${{ env.UBUNTU_VERSION }}-${{ steps.system-info.outputs.CPU_ARCH }}-rocm-${{ matrix.ROCM_VERSION }}.zip ./build/bin ./start-server.sh ./start-server.bat
 
       - name: Upload artifacts
         if: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/master' ) || github.event.inputs.create_release == 'true' }}
diff --git a/.github/workflows/unsloth-sd-prebuilt.yml b/.github/workflows/unsloth-sd-prebuilt.yml
index cd981b98f4..e6f582e329 100644
--- a/.github/workflows/unsloth-sd-prebuilt.yml
+++ b/.github/workflows/unsloth-sd-prebuilt.yml
@@ -1,26 +1,29 @@
 # SPDX-License-Identifier: MIT
 # Copyright 2026-present the Unsloth AI Inc. team.
 
-name: Unsloth SD prebuilt (CPU/Apple/CUDA/Vulkan)
+name: Unsloth SD prebuilt (CPU/Apple/CUDA/Vulkan/ROCm)
 
 # Build and publish our stable-diffusion.cpp (sd-cli + sd-server) prebuilts for
-# CPU, Apple, CUDA, and Vulkan. MiniMax-H3 uses the native engine on GPU hosts too.
+# CPU, Apple, CUDA, Vulkan, and ROCm. MiniMax-H3 uses the native engine on GPU hosts too.
 # Vulkan must carry the same H3 and graph-segment fixes as the other backends,
 # rather than sending AMD hosts to an older upstream fallback binary.
 #
 # Shape mirrors unslothai/llama.cpp's prebuilt pipeline, simplified for a 7-way
-# CPU/Apple/Vulkan matrix plus one best-effort CUDA leg:
+# CPU/Apple/Vulkan matrix plus best-effort CUDA and ROCm legs:
 #   resolve      -- take THIS repository at the requested ref, merge the pinned PR set
 #                   in scripts/unsloth/pr-set.json, stamp a source tarball, name it
 #                   after the upstream release the tree descends from (supply-chain
 #                   aged), decide if our release already exists.
 #   build-unix   -- macOS arm64 (Metal) + x64, Linux x64 + arm64, Linux Vulkan.
 #   build-windows-- Windows x64 CPU + Vulkan (MSVC + Ninja).
-#   build-linux-cuda -- Linux x64, CUDA 12.8, best effort (continue-on-error).
+#   build-linux-cuda / build-windows-cuda -- x64, CUDA 12.8 and 13.3, best effort
+#                   (continue-on-error); the CUDA runtime is not bundled, see build-linux-cuda.
+#   build-linux-rocm -- Linux x64, ROCm 7.14.0 with its userspace bundled, best effort
+#                   like CUDA. Without it AMD Linux hosts fall back to upstream or CPU (unsloth#8814).
 #   assemble     -- fingerprint gate, sha256 + manifest, coverage gate, atomic
 #                   draft->publish. If any CPU/Apple/Vulkan leg fails, assemble
 #                   is skipped and nothing is published (the Studio needs the full
-#                   asset set). The CUDA leg is exempt: it ships when it built and is
+#                   asset set). The CUDA and ROCm legs are exempt: each ships when it built and is
 #                   absent when it did not, so it can never hold back the fallbacks.
 #
 # Asset names match the Studio installer's resolve_release_asset.
@@ -220,6 +223,12 @@ jobs:
           # (we build with SD_SERVER_BUILD_FRONTEND / SD_WEBP / SD_WEBM off), so fetch only
           # ggml to keep the source tarball small and the build self-contained.
           git submodule update --init --recursive --depth 1 ggml
+          # ggml changes we carry on top of the pinned submodule commit (the submodule points at
+          # leejet/ggml, which we do not push to). A patch that no longer applies fails the build.
+          for p in scripts/unsloth/ggml-patches/*.patch; do
+            [ -e "$p" ] || continue
+            git -C ggml apply --verbose "../$p"
+          done
           COMMIT="$(git rev-parse HEAD)"
           SRC_ARTIFACT="sd-source-${TAG}"
           if [ "$EXISTS" != "true" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
@@ -259,23 +268,29 @@ jobs:
           - label: Darwin-macOS-arm64
             runner: macos-14
             arch: arm64
-            defines: "-DSD_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0"
+            # Static: apple_m1 variant drops FP16_VA (~40% slower). BLAS explicit: sd.cpp routes big matmuls to it.
+            defines: "-DSD_METAL=ON -DGGML_METAL_EMBED_LIBRARY=ON -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Apple -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0"
             gate: "true"
           - label: Darwin-macOS-x86_64
             runner: macos-15-intel
             arch: x86_64
-            defines: "-DSD_METAL=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"
+            # BLAS off: its matmul path was only measured on Apple Silicon.
+            defines: "-DSD_METAL=OFF -DGGML_BLAS=OFF -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3"
             gate: "true"
+            cpu_floor: x64
           - label: Linux-Ubuntu-22.04-x86_64
             runner: ubuntu-22.04
             arch: x86_64
             defines: ""
             gate: "false"
+            cpu_floor: x64
           - label: Linux-Ubuntu-24.04-aarch64
             runner: ubuntu-24.04-arm
             arch: aarch64
-            defines: ""
+            # GCC 13 (the image default) rejects +sme, which the Armv9.2 variants need.
+            defines: "-DCMAKE_C_COMPILER=gcc-14 -DCMAKE_CXX_COMPILER=g++-14"
             gate: "false"
+            cpu_floor: armv8.0_1
           - label: Linux-Ubuntu-22.04-x86_64-vulkan
             runner: ubuntu-22.04
             arch: x86_64
@@ -321,20 +336,38 @@ jobs:
           echo "VULKAN_SDK=$sdk_root/x86_64" >> "$GITHUB_ENV"
           echo "$sdk_root/x86_64/bin" >> "$GITHUB_PATH"
 
+      # CPU bundles ship all ggml CPU variants, picked per host at startup; -DGGML_NATIVE=OFF alone
+      # pins AVX2 (SIGILL on older x86) or plain Armv8.0. Metal/Vulkan stay static.
       - name: Build sd-cli + sd-server
         working-directory: src
         run: |
           set -euo pipefail
+          CPU_VARIANTS=""
+          if [ -n "${{ matrix.cpu_floor }}" ]; then
+            CPU_VARIANTS="-DSD_BUILD_SHARED_GGML_LIB=ON -DBUILD_SHARED_LIBS=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON"
+            # macOS: @loader_path only, never an absolute rpath into the build tree.
+            if [ "$(uname -s)" = "Darwin" ]; then
+              CPU_VARIANTS="$CPU_VARIANTS -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_RPATH=@loader_path"
+            fi
+          fi
           cmake -B build \
             -DCMAKE_BUILD_TYPE=Release \
             -DSD_BUILD_EXAMPLES=ON \
             -DSD_SERVER_BUILD_FRONTEND=OFF \
             -DSD_WEBP=OFF -DSD_WEBM=OFF \
             -DGGML_NATIVE=OFF \
+            $CPU_VARIANTS \
             ${{ matrix.defines }}
           if [ "$(uname -s)" = "Darwin" ]; then J="$(sysctl -n hw.logicalcpu)"; else J="$(nproc)"; fi
           cmake --build build --config Release -j "$J" --target sd-cli sd-server
 
+      - name: Pin the bundle's own ggml (CPU variants)
+        if: ${{ matrix.cpu_floor != '' && runner.os == 'Linux' }}
+        run: |
+          set -euo pipefail
+          sudo apt-get update -qq && sudo apt-get install -y -qq patchelf
+          bash tooling/scripts/unsloth/cpu_variants_gate.sh pin src/build/bin
+
       - name: macOS load gate
         if: ${{ matrix.gate == 'true' }}
         run: |
@@ -354,6 +387,10 @@ jobs:
           LICENSE_FILE: ${{ github.workspace }}/src/LICENSE
         run: python3 tooling/scripts/unsloth/package_bundle.py
 
+      - name: CPU variants gate
+        if: ${{ matrix.cpu_floor != '' }}
+        run: bash tooling/scripts/unsloth/cpu_variants_gate.sh check "dist/sd-${{ needs.resolve.outputs.tag }}-bin-${{ matrix.label }}.zip" "${{ matrix.cpu_floor }}"
+
       - name: Upload bundle
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         with:
@@ -368,21 +405,38 @@ jobs:
   # against 21.5 s/step at 960x544 from a CUDA build. That is the difference between a render
   # taking four hours and taking eleven minutes.
   #
+  # One bundle per CUDA major, and no CUDA runtime in any of them. Studio always installs
+  # unslothai/llama.cpp, whose CUDA build links the same cudart / cublas / cublasLt and already
+  # resolves them on every CUDA host (the venv's torch and nvidia-* wheels, CUDA_PATH, the paired
+  # cudart archive on Windows). Studio picks the bundle whose major matches that llama.cpp build
+  # and hands sd-cli the same directories, so the toolkits here match llama.cpp's cuda12 / cuda13
+  # lines. ggml is a shared library so sd-cli and sd-server share one copy of the CUDA kernels.
+  #
   # continue-on-error, and deliberately so. The CPU and Apple assets are what Studio falls back
   # to, and a CUDA toolchain failure must never stop them publishing. assemble collects bundles
   # by the sd-*-bin-* pattern, so this asset appears when it built and is simply absent when it
   # did not; the coverage gate requires CPU/Apple/Vulkan assets but leaves CUDA optional.
   build-linux-cuda:
-    name: Linux-Ubuntu-22.04-x86_64-cuda12
+    name: Linux-Ubuntu-22.04-x86_64-${{ matrix.line }}
     needs: resolve
     continue-on-error: true
     if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }}
     runs-on: ubuntu-22.04
-    # One source of truth for both: the ccache key has to track whatever changes
-    # the objects, and these two do.
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - line: cuda12
+            cuda: "12.8.0"
+            archs: "70;75;80;86;89;90;100;120"
+          # CUDA 13 dropped Volta.
+          - line: cuda13
+            cuda: "13.3"
+            archs: "75;80;86;89;90;100;120"
+    # The ccache key tracks both, since both change the objects.
     env:
-      CUDA_VERSION: "12.8.1"
-      CUDA_ARCHS: "70;75;80;86;89;90;100;120"
+      CUDA_VERSION: ${{ matrix.cuda }}
+      CUDA_ARCHS: ${{ matrix.archs }}
     steps:
       - name: Checkout mirror (tooling)
         uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
@@ -409,18 +463,36 @@ jobs:
           mkdir -p src
           tar -xzf "srcpkg/sd-source-${{ needs.resolve.outputs.tag }}.tar.gz" -C src
 
-      - name: Install CUDA toolkit
-        id: cuda-toolkit
-        uses: Jimver/cuda-toolkit@95b5a29a2823ef87666dcd45ec236fc4f82498d0  # v0.2.22
-        with:
-          cuda: ${{ env.CUDA_VERSION }}
-          method: "network"
-          # sub-packages are installed as cuda-<name>-12-8. cuBLAS is not under that
-          # prefix (it ships as libcublas / libcublas-dev), so it has to go in the
-          # other list or apt cannot find it. cudart-dev is what carries the headers;
-          # cudart alone is the runtime and does not compile anything.
-          sub-packages: '["nvcc", "cudart", "cudart-dev", "thrust"]'
-          non-cuda-sub-packages: '["libcublas", "libcublas-dev"]'
+      # Component tables copied from unslothai/llama.cpp's unsloth-cuda-linux-setup, so both projects
+      # link the sonames Studio already resolves for llama-server. nvrtc: cudnn-frontend's headers
+      # include nvrtc.h (it dlopens libnvrtc itself, so nothing links it).
+      - name: Install CUDA toolkit ${{ env.CUDA_VERSION }} (NVIDIA redist)
+        run: |
+          set -eux
+          case "$CUDA_VERSION" in
+            12.8.0) COMPONENTS="cuda_cudart:12.8.90 cuda_nvcc:12.8.93 cuda_nvrtc:12.8.93 libcublas:12.8.4.1 cuda_nvtx:12.8.90 cuda_profiler_api:12.8.90 cuda_cccl:12.8.90" ;;
+            13.3)   COMPONENTS="cuda_crt:13.3.33 cuda_cudart:13.3.29 cuda_nvcc:13.3.33 cuda_nvrtc:13.3.33 libcublas:13.5.1.27 libnvvm:13.3.33 cuda_nvtx:13.3.29 cuda_profiler_api:13.3.27 cccl:13.3.3.3.1" ;;
+            *) echo "no redist component table for CUDA $CUDA_VERSION" >&2; exit 1 ;;
+          esac
+          PREFIX="$HOME/cuda-$CUDA_VERSION"
+          mkdir -p "$PREFIX"
+          BASE="https://developer.download.nvidia.com/compute/cuda/redist"
+          for cv in $COMPONENTS; do
+            comp="${cv%:*}"; ver="${cv#*:}"
+            name="${comp}-linux-x86_64-${ver}-archive"
+            curl -fsSL --retry 3 -o comp.tar.xz "$BASE/${comp}/linux-x86_64/${name}.tar.xz"
+            tar -xf comp.tar.xz
+            cp -a "${name}/." "$PREFIX"/
+            rm -rf comp.tar.xz "${name}"
+          done
+          ln -sfn lib "$PREFIX/lib64"
+          {
+            echo "CUDA_PATH=$PREFIX"
+            echo "CUDA_HOME=$PREFIX"
+            echo "LD_LIBRARY_PATH=$PREFIX/lib:${LD_LIBRARY_PATH:-}"
+          } >> "$GITHUB_ENV"
+          echo "$PREFIX/bin" >> "$GITHUB_PATH"
+          "$PREFIX/bin/nvcc" --version
 
       # This leg rebuilt every object on every run: 3903 s of the 4121 s job on
       # 2026-08-09, and 4942 s on the run before it, with no speedup between the
@@ -450,17 +522,40 @@ jobs:
           # keeps what it compiled.
           save: false
 
+      - name: cuDNN headers (build time only)
+        run: |
+          set -euo pipefail
+          # ggml-cuda's cuDNN attention path (GGML_CUDA_CUDNN) compiles against cudnn.h, but the
+          # library itself is opened with dlopen when the first attention op runs and is NOT
+          # bundled: a host without a loadable libcudnn.so.9 keeps the ggml attention kernels.
+          # Only the headers package is fetched (80 KB), pinned by version and checksum.
+          deb=libcudnn9-headers-cuda-12_9.27.0.42-1_amd64.deb
+          curl -fsSL --retry 3 -o "$RUNNER_TEMP/$deb" \
+            "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/$deb"
+          echo "68243ab04422da565e0bafec00f64ac08cfd82d805c8015c0c503af7ab4dd1d6  $RUNNER_TEMP/$deb" | sha256sum -c -
+          dpkg -x "$RUNNER_TEMP/$deb" "$RUNNER_TEMP/cudnn-deb"
+          # The package ships cudnn*_v9.h; the unversioned names normally come from the -dev
+          # package's alternatives, so link them here.
+          inc="$RUNNER_TEMP/cudnn-include"
+          mkdir -p "$inc"
+          for f in "$RUNNER_TEMP"/cudnn-deb/usr/include/x86_64-linux-gnu/cudnn*_v9.h; do
+            b="$(basename "$f")"
+            ln -s "$f" "$inc/${b%_v9.h}.h"
+          done
+          test -e "$inc/cudnn.h"
+          echo "CUDNN_INCLUDE_DIR=$inc" >> "$GITHUB_ENV"
+
       - name: Build sd-cli + sd-server (CUDA)
         working-directory: src
         run: |
           set -euo pipefail
-          # Volta through Blackwell. V100 can run H3 with quantized weights and CPU offload,
-          # but a device probe succeeds even when the binary has no sm_70 kernels. Include
-          # sm_70 so generation does not fail at the first CUDA kernel launch.
+          # cuda12 covers Volta through Blackwell. V100 can run H3 with quantized weights and CPU
+          # offload, but a device probe succeeds even when the binary has no sm_70 kernels, so
+          # sm_70 stays in the list.
           #
-          # The CUDA launcher matters more than the other two here: Jimver installs nvcc outside
-          # the default search, and nvcc is nearly the whole build, so caching only C/CXX would
-          # leave the expensive half uncached.
+          # The CUDA launcher matters more than the other two here: nvcc lives outside the default
+          # search, and nvcc is nearly the whole build, so caching only C/CXX would leave the
+          # expensive half uncached.
           cmake -B build \
             -DCMAKE_BUILD_TYPE=Release \
             -DSD_BUILD_EXAMPLES=ON \
@@ -468,6 +563,11 @@ jobs:
             -DSD_WEBP=OFF -DSD_WEBM=OFF \
             -DGGML_NATIVE=OFF \
             -DSD_CUDA=ON \
+            -DSD_BUILD_SHARED_GGML_LIB=ON \
+            -DBUILD_SHARED_LIBS=ON \
+            -DCMAKE_CUDA_COMPILER="$CUDA_PATH/bin/nvcc" \
+            -DGGML_CUDA_CUDNN=ON \
+            -DGGML_CUDA_CUDNN_INCLUDE_DIR="$CUDNN_INCLUDE_DIR" \
             -DCMAKE_CUDA_ARCHITECTURES="$CUDA_ARCHS" \
             -DCMAKE_C_COMPILER_LAUNCHER=ccache \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
@@ -476,43 +576,66 @@ jobs:
           # Diagnostic only, under set -e: never fail a good build over stats.
           ccache --show-stats || true
 
-      - name: Bundle the CUDA runtime beside the binaries
+      - name: Pin the bundle's own ggml; leave the CUDA runtime to the host
         run: |
           set -euo pipefail
-          # A Studio host has an NVIDIA driver but not necessarily a CUDA runtime, and Studio's
-          # torch install keeps its own copies somewhere we must not depend on. Ship ours and
-          # point the binaries at their own directory, or sd-cli dies on a missing libcublas.
           BIN="${GITHUB_WORKSPACE}/src/build/bin"
-          CUDA_ROOT="${{ steps.cuda-toolkit.outputs.CUDA_PATH }}"
+          MAJOR="${CUDA_VERSION%%.*}"
           sudo apt-get update -qq && sudo apt-get install -y -qq patchelf
-          # cudart lands under the toolkit root; the libcublas debs land in the system
-          # multiarch dir. Search both rather than guess which is which.
-          for so in libcudart.so.12 libcublas.so.12 libcublasLt.so.12; do
-            src="$(find "$CUDA_ROOT" /usr/lib/x86_64-linux-gnu -name "${so}*" -type f 2>/dev/null | sort | tail -1)"
-            [ -n "$src" ] || { echo "ERROR: $so not found under $CUDA_ROOT or /usr/lib/x86_64-linux-gnu" >&2; exit 1; }
-            cp -L "$src" "$BIN/$so"
+          # DT_RPATH, not RUNPATH: it is searched before LD_LIBRARY_PATH and covers the whole
+          # dependency chain, so this bundle's libggml*.so win over llama.cpp's same-named ones
+          # when Studio puts both directories on the loader path.
+          for f in "$BIN"/sd-cli "$BIN"/sd-server "$BIN"/libggml*.so* "$BIN"/libstable-diffusion*.so*; do
+            [ -f "$f" ] && [ ! -L "$f" ] || continue
+            patchelf --force-rpath --set-rpath '$ORIGIN' "$f"
           done
-          for exe in sd-cli sd-server; do
-            patchelf --set-rpath '$ORIGIN' "$BIN/$exe"
+          ls -la "$BIN"
+          if ls "$BIN" | grep -qE '^lib(cudart|cublas|cublasLt)\.so'; then
+            echo "ERROR: a CUDA runtime library is in the bundle; the host provides it" >&2
+            exit 1
+          fi
+          cuda_lib="$(ls "$BIN"/libggml-cuda.so* | head -1)"
+          for so in "libcudart.so.${MAJOR}" "libcublas.so.${MAJOR}"; do
+            readelf -d "$cuda_lib" | grep -qF "[${so}]" \
+              || { echo "ERROR: $(basename "$cuda_lib") does not link ${so}" >&2; exit 1; }
+          done
+          # The host supplies exactly what llama.cpp's build of this major links. cuDNN and NVRTC are
+          # run-time dlopens only; linking them would stop the binaries loading on hosts without them.
+          allowed=" libcuda.so.1 libcudart.so.${MAJOR} libcublas.so.${MAJOR} libcublasLt.so.${MAJOR} "
+          for f in "$BIN"/sd-cli "$BIN"/sd-server "$BIN"/libggml*.so* "$BIN"/libstable-diffusion*.so*; do
+            [ -f "$f" ] && [ ! -L "$f" ] || continue
+            for so in $(readelf -d "$f" | sed -n 's/.*(NEEDED).*\[\(lib\(cu\|nv\)[^]]*\)\].*/\1/p'); do
+              case "$allowed" in
+                *" $so "*) ;;
+                *) echo "ERROR: $(basename "$f") links $so; only ${allowed} may come from the host" >&2; exit 1 ;;
+              esac
+            done
           done
-          ldd "$BIN/sd-cli" | sed -n '1,40p'
 
       - name: Package bundle
         env:
           BIN_DIR: ${{ github.workspace }}/src/build/bin
           OUT_DIR: ${{ github.workspace }}/dist
           TAG: ${{ needs.resolve.outputs.tag }}
-          LABEL: Linux-Ubuntu-22.04-x86_64-cuda12
+          LABEL: Linux-Ubuntu-22.04-x86_64-${{ matrix.line }}
           COMMIT: ${{ needs.resolve.outputs.commit }}
           SOURCE_REPO: ${{ github.repository }}
-          LICENSE_FILE: ${{ github.workspace }}/src/LICENSE
-        run: python3 tooling/scripts/unsloth/package_bundle.py
+          LICENSE_FILE: ${{ runner.temp }}/LICENSE
+        run: |
+          set -euo pipefail
+          # The binaries compile in cudnn-frontend's header-only templates, so its MIT notice ships too.
+          {
+            cat src/LICENSE
+            printf '\n\n---- cudnn-frontend (https://github.com/NVIDIA/cudnn-frontend) ----\n\n'
+            cat src/build/_deps/ggml_cudnn_frontend-src/LICENSE.txt
+          } > "$LICENSE_FILE"
+          python3 tooling/scripts/unsloth/package_bundle.py
 
       - name: Upload bundle
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         with:
-          name: sd-${{ needs.resolve.outputs.tag }}-bin-Linux-Ubuntu-22.04-x86_64-cuda12
-          path: dist/sd-${{ needs.resolve.outputs.tag }}-bin-Linux-Ubuntu-22.04-x86_64-cuda12.zip
+          name: sd-${{ needs.resolve.outputs.tag }}-bin-Linux-Ubuntu-22.04-x86_64-${{ matrix.line }}
+          path: dist/sd-${{ needs.resolve.outputs.tag }}-bin-Linux-Ubuntu-22.04-x86_64-${{ matrix.line }}.zip
           if-no-files-found: error
           retention-days: 7
 
@@ -538,6 +661,200 @@ jobs:
           # restored from, so the save is never a no-op against itself.
           key: ccache-sd-cuda-${{ env.CUDA_VERSION }}-${{ steps.cckey.outputs.archs }}-${{ needs.resolve.outputs.tag }}-
 
+  # Windows CUDA, same runtime policy as build-linux-cuda: the ggml*.dll ship beside the executables
+  # (Windows searches the executable's own directory first, so llama.cpp's same-named DLLs elsewhere
+  # on PATH never win), and cudart64 / cublas64 / cublasLt64 come from the host exactly as they do
+  # for unslothai/llama.cpp's Windows CUDA build. No cuDNN attention path: the ggml kernels serve.
+  build-windows-cuda:
+    name: win-${{ matrix.line }}-x64
+    needs: resolve
+    continue-on-error: true
+    if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }}
+    runs-on: windows-2022
+    timeout-minutes: 300
+    environment: release-signing
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - line: cuda12
+            cuda: "12.8.0"
+            archs: "70;75;80;86;89;90;100;120"
+          - line: cuda13
+            cuda: "13.3"
+            archs: "75;80;86;89;90;100;120"
+    defaults:
+      run:
+        shell: pwsh
+    steps:
+      - name: Checkout mirror (tooling)
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
+        with:
+          path: tooling
+          fetch-depth: 1
+
+      - name: Download source @ ${{ needs.resolve.outputs.tag }}
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: ${{ needs.resolve.outputs.source_artifact }}
+          path: srcpkg
+
+      - name: Extract source
+        shell: bash
+        run: |
+          set -eux
+          mkdir -p src
+          tar -xzf "srcpkg/sd-source-${{ needs.resolve.outputs.tag }}.tar.gz" -C src
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+        with:
+          python-version: "3.11"
+      - name: Setup MSVC
+        uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756  # v1.13.0
+
+      # Component tables copied from unslothai/llama.cpp's unsloth-cuda-windows-setup.
+      - name: Install CUDA toolkit ${{ matrix.cuda }} (NVIDIA redist)
+        run: |
+          $ErrorActionPreference = 'Stop'
+          switch ("${{ matrix.cuda }}") {
+            '12.8.0' { $components = @('cuda_cudart:12.8.57', 'cuda_nvcc:12.8.61', 'cuda_nvrtc:12.8.61', 'libcublas:12.8.3.14', 'cuda_nvtx:12.8.55', 'cuda_profiler_api:12.8.55', 'cuda_cccl:12.8.55', 'visual_studio_integration:12.8.55') }
+            '13.3'   { $components = @('cuda_crt:13.3.33', 'cuda_cudart:13.3.29', 'cuda_nvcc:13.3.33', 'cuda_nvrtc:13.3.33', 'libcublas:13.5.1.27', 'libnvvm:13.3.33', 'cuda_nvtx:13.3.29', 'cuda_profiler_api:13.3.27', 'cccl:13.3.3.3.1', 'visual_studio_integration:13.3.27') }
+            default  { throw "no redist component table for CUDA ${{ matrix.cuda }}" }
+          }
+          $line = "${{ matrix.cuda }}" -replace '^(\d+\.\d+).*$', '$1'
+          $prefix = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$line"
+          $stage = Join-Path $env:RUNNER_TEMP 'cuda-redist'
+          New-Item -ItemType Directory -Force -Path $prefix, $stage | Out-Null
+          foreach ($cv in $components) {
+            $comp, $ver = $cv.Split(':')
+            $name = "$comp-windows-x86_64-$ver-archive"
+            $zip = Join-Path $stage "$name.zip"
+            curl.exe -fsSL --retry 3 -o $zip "https://developer.download.nvidia.com/compute/cuda/redist/$comp/windows-x86_64/$name.zip"
+            if ($LASTEXITCODE -ne 0) { throw "download failed: $name" }
+            tar.exe -xf $zip -C $stage
+            if ($LASTEXITCODE -ne 0) { throw "extract failed: $name" }
+            robocopy (Join-Path $stage $name) $prefix /E /NFL /NDL /NJH /NJS /NP | Out-Null
+            if ($LASTEXITCODE -ge 8) { throw "copy failed: $name ($LASTEXITCODE)" }
+            Remove-Item -Recurse -Force $zip, (Join-Path $stage $name)
+          }
+          "$prefix\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          "CUDA_PATH=$prefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          "CUDA_HOME=$prefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          "CUDA_PATH_V$($line.Replace('.', '_'))=$prefix" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+          exit 0
+
+      - name: ccache key
+        id: cckey
+        shell: bash
+        run: echo "archs=$(echo '${{ matrix.archs }}' | tr ';' '-')" >> "$GITHUB_OUTPUT"
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03  # v1.2.23
+        with:
+          key: sd-cuda-${{ matrix.cuda }}-windows-${{ steps.cckey.outputs.archs }}-${{ needs.resolve.outputs.tag }}
+          restore-keys: |
+            sd-cuda-${{ matrix.cuda }}-windows-${{ steps.cckey.outputs.archs }}
+          append-timestamp: false
+          variant: ccache
+          max-size: 3G
+          save: false
+
+      - name: Build sd-cli + sd-server (CUDA)
+        working-directory: src
+        run: |
+          nvcc --version
+          cmake -S . -B build -G Ninja `
+            -DCMAKE_BUILD_TYPE=Release `
+            -DCMAKE_CXX_FLAGS='/bigobj' `
+            -DSD_BUILD_EXAMPLES=ON `
+            -DSD_SERVER_BUILD_FRONTEND=OFF `
+            -DSD_WEBP=OFF -DSD_WEBM=OFF `
+            -DGGML_NATIVE=OFF `
+            -DSD_CUDA=ON `
+            -DSD_BUILD_SHARED_GGML_LIB=ON `
+            -DBUILD_SHARED_LIBS=ON `
+            -DCMAKE_CUDA_ARCHITECTURES='${{ matrix.archs }}' `
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache `
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache `
+            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          # -j 3: a multi-arch nvcc TU peaks near 3 GB RSS on the 16 GB hosted runner.
+          cmake --build build --config Release -j 3 --target sd-cli sd-server
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          ccache -s
+          exit 0
+
+      - name: Leave the CUDA runtime to the host
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $bin = Join-Path $env:GITHUB_WORKSPACE 'src/build/bin'
+          $major = "${{ matrix.cuda }}".Split('.')[0]
+          Get-ChildItem $bin | Format-Table Name, Length
+          $bundled = @(Get-ChildItem $bin -Filter *.dll | Where-Object { $_.Name -match '^(cudart|cublas|cublasLt)64_' })
+          if ($bundled.Count -gt 0) { throw "CUDA runtime DLLs in the bundle: $($bundled.Name -join ', ')" }
+          $cuda = Join-Path $bin 'ggml-cuda.dll'
+          if (-not (Test-Path $cuda)) { throw 'ggml-cuda.dll was not built' }
+          $imports = (& dumpbin /dependents $cuda) -join "`n"
+          $imports
+          # cublas is always the host's. cudart is too on CUDA 12; CUDA 13 links it statically, as in
+          # unslothai/llama.cpp's cuda13 build, which imports cublas64_13.dll alone.
+          if ($imports -notmatch [regex]::Escape("cublas64_$major.dll")) { throw "ggml-cuda.dll does not import cublas64_$major.dll" }
+          # Anything else CUDA-shaped (another major, nvrtc, cudnn) would not be on the host.
+          $allowed = @('nvcuda.dll', "cudart64_$major.dll", "cublas64_$major.dll", "cublasLt64_$major.dll")
+          foreach ($pe in Get-ChildItem $bin -Include *.dll, *.exe -Recurse) {
+            $deps = [regex]::Matches(((& dumpbin /dependents $pe.FullName) -join "`n"), '(?im)^\s*((?:cu|nv)\S*\.dll)\s*$') | ForEach-Object { $_.Groups[1].Value }
+            $extra = @($deps | Where-Object { $allowed -notcontains $_ })
+            if ($extra.Count -gt 0) { throw "$($pe.Name) imports $($extra -join ', '); only $($allowed -join ', ') may come from the host" }
+          }
+
+      - name: Sign Windows binaries
+        uses: ./tooling/.github/actions/sign-windows
+        with:
+          path: src/build/bin
+          azure-client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          azure-client-secret: ${{ secrets.AZURE_CLIENT_SECRET }}
+          azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          azure-account: ${{ secrets.AZURE_TRUSTED_SIGNING_ACCOUNT_NAME }}
+          azure-certificate-profile: ${{ secrets.AZURE_CERTIFICATE_PROFILE_NAME }}
+
+      - name: Package bundle
+        env:
+          BIN_DIR: ${{ github.workspace }}/src/build/bin
+          OUT_DIR: ${{ github.workspace }}/dist
+          TAG: ${{ needs.resolve.outputs.tag }}
+          LABEL: win-${{ matrix.line }}-x64
+          COMMIT: ${{ needs.resolve.outputs.commit }}
+          SOURCE_REPO: ${{ github.repository }}
+          LICENSE_FILE: ${{ github.workspace }}/src/LICENSE
+        run: python tooling/scripts/unsloth/package_bundle.py
+
+      - name: Verify every PE in the bundle is signed
+        run: |
+          $ErrorActionPreference = 'Continue'
+          $zips = @(Get-ChildItem dist -Filter *.zip -ErrorAction SilentlyContinue | ForEach-Object { $_.FullName })
+          if ($zips.Count -eq 0) {
+            Write-Host '::error::no bundle in dist/; packaging produced nothing to verify'
+            exit 1
+          }
+          & tooling/.github/scripts/assert-windows-bundle-signed.ps1 -Path $zips
+          exit $LASTEXITCODE
+
+      - name: Upload bundle
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: sd-${{ needs.resolve.outputs.tag }}-bin-win-${{ matrix.line }}-x64
+          path: dist/sd-${{ needs.resolve.outputs.tag }}-bin-win-${{ matrix.line }}-x64.zip
+          if-no-files-found: error
+          retention-days: 7
+
+      - name: Save ccache
+        # always(): a job killed by the cap takes the cancellation path, the most expensive case to lose.
+        if: ${{ always() }}
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: ${{ github.workspace }}\.ccache
+          key: ccache-sd-cuda-${{ matrix.cuda }}-windows-${{ steps.cckey.outputs.archs }}-${{ needs.resolve.outputs.tag }}-
+
   build-windows:
     name: ${{ matrix.label }}
     needs: resolve
@@ -549,6 +866,7 @@ jobs:
         include:
           - label: win-cpu-x64
             vulkan: "OFF"
+            cpu_floor: x64
           - label: win-vulkan-x64
             vulkan: "ON"
     env:
@@ -608,6 +926,10 @@ jobs:
         shell: pwsh
         working-directory: src
         run: |
+          $cpuVariants = @()
+          if ('${{ matrix.cpu_floor }}' -ne '') {
+            $cpuVariants = @('-DSD_BUILD_SHARED_GGML_LIB=ON', '-DBUILD_SHARED_LIBS=ON', '-DGGML_BACKEND_DL=ON', '-DGGML_CPU_ALL_VARIANTS=ON')
+          }
           cmake -S . -B build -G Ninja `
             -DCMAKE_BUILD_TYPE=Release `
             -DCMAKE_CXX_FLAGS='/bigobj' `
@@ -615,7 +937,9 @@ jobs:
             -DSD_SERVER_BUILD_FRONTEND=OFF `
             -DSD_WEBP=OFF -DSD_WEBM=OFF `
             -DSD_VULKAN=${{ matrix.vulkan }} `
-            -DGGML_NATIVE=OFF
+            -DGGML_NATIVE=OFF `
+            @cpuVariants
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
           cmake --build build --config Release -j 3 --target sd-cli sd-server
 
       # Before packaging, so every PE the bundle carries is signed. Windows
@@ -661,6 +985,44 @@ jobs:
           & tooling/.github/scripts/assert-windows-bundle-signed.ps1 -Path $zips
           exit $LASTEXITCODE
 
+      - name: CPU variants gate
+        if: ${{ matrix.cpu_floor != '' }}
+        shell: pwsh
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $zip = (Get-ChildItem dist -Filter *.zip | Select-Object -First 1).FullName
+          $work = Join-Path $env:RUNNER_TEMP 'cpu-variants'
+          $empty = Join-Path $env:RUNNER_TEMP 'cpu-variants-cwd'
+          Remove-Item -Recurse -Force $work, $empty -ErrorAction SilentlyContinue
+          New-Item -ItemType Directory -Force $work, $empty | Out-Null
+          Expand-Archive -Path $zip -DestinationPath $work
+          $cli = Get-ChildItem $work -Recurse -Filter sd-cli.exe | Select-Object -First 1
+          $dir = $cli.DirectoryName
+          $variants = @(Get-ChildItem $dir -Filter 'ggml-cpu-*.dll' | ForEach-Object { $_.BaseName -replace '^ggml-cpu-', '' })
+          Write-Host "variants: $($variants -join ' ')"
+          if ($variants -notcontains '${{ matrix.cpu_floor }}') {
+            Write-Host "::error::floor variant ${{ matrix.cpu_floor }} missing from the bundle"
+            exit 1
+          }
+          Push-Location $empty
+          $ErrorActionPreference = 'Continue'
+          $out = (& $cli.FullName -m (Join-Path $empty 'none.gguf') -p x -v 2>&1 | Out-String)
+          Pop-Location
+          $m = [regex]::Match($out, 'loaded CPU backend from (.+?\.dll)')
+          if (-not $m.Success) {
+            Write-Host $out
+            Write-Host '::error::sd-cli loaded no CPU backend variant'
+            exit 1
+          }
+          $loaded = $m.Groups[1].Value.Trim()
+          Write-Host "loaded: $loaded"
+          if ((Split-Path -Parent ([IO.Path]::GetFullPath($loaded))) -ne [IO.Path]::GetFullPath($dir)) {
+            Write-Host "::error::variant came from outside the bundle: $loaded"
+            exit 1
+          }
+          # sd-cli itself exits 1 at the missing model; that is the expected stop, not the verdict.
+          exit 0
+
       - name: Upload bundle
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
         with:
@@ -669,9 +1031,185 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
+  # Linux ROCm from TheRock wheels like upstream's build.yml, but the linked ROCm userspace ships
+  # in the bundle with rpath $ORIGIN: upstream's zip dies on a host whose ROCm differs or is
+  # absent (unsloth#8814). Only the kernel driver comes from the host.
+  build-linux-rocm:
+    name: Linux-Ubuntu-24.04-x86_64-rocm-7.14.0
+    needs: resolve
+    continue-on-error: true
+    if: ${{ needs.resolve.outputs.exists != 'true' || github.event_name == 'workflow_dispatch' }}
+    runs-on: ubuntu-24.04
+    env:
+      ROCM_VERSION: "7.14.0"
+      # Consumer parts, upstream's Windows list. CDNA hosts run diffusers and never reach this.
+      GPU_TARGETS: "gfx1010;gfx1011;gfx1012;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201"
+      # rocm-sdk re-expands clang from the wheel every run, so its mtime always differs and
+      # the default compiler_check=mtime misses every object.
+      CCACHE_COMPILERCHECK: content
+    steps:
+      - name: Checkout mirror (tooling)
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4.4.0
+        with:
+          path: tooling
+          fetch-depth: 1
+
+      - name: Free disk for the ROCm wheels
+        run: |
+          set -eux
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup \
+            /usr/local/share/boost /usr/share/swift /opt/hostedtoolcache
+          docker system prune -af || true
+          df -h /
+
+      - name: Download source @ ${{ needs.resolve.outputs.tag }}
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093  # v4.3.0
+        with:
+          name: ${{ needs.resolve.outputs.source_artifact }}
+          path: srcpkg
+      - name: Extract source
+        run: |
+          set -eux
+          mkdir -p src
+          tar -xzf "srcpkg/sd-source-${{ needs.resolve.outputs.tag }}.tar.gz" -C src
+
+      - name: Install ROCm (TheRock wheels)
+        run: |
+          set -euo pipefail
+          sudo apt-get update -qq && sudo apt-get install -y -qq ninja-build patchelf
+          python3 -m venv .rocm
+          . .rocm/bin/activate
+          python -m pip install -q --upgrade pip
+          # libraries/devel carry no GPU code: rocBLAS/hipBLASLt kernels and kpack archives come
+          # from one rocm-sdk-device-<gfx> wheel per target, linked into the devel tree by
+          # `rocm-sdk path`.
+          DEVICE="$(echo "$GPU_TARGETS" | sed 's/;/,device-/g; s/^/device-/')"
+          python -m pip install -q --no-cache-dir --index-url https://repo.amd.com/rocm/whl-multi-arch/ \
+            "rocm[libraries,devel,${DEVICE}]==${ROCM_VERSION}"
+          ROCM_PATH="$(rocm-sdk path --root)"
+          {
+            echo "ROCM_PATH=$ROCM_PATH"
+            echo "HIP_PATH=$ROCM_PATH"
+            echo "CMAKE_PREFIX_PATH=$(rocm-sdk path --cmake)"
+            echo "LD_LIBRARY_PATH=$ROCM_PATH/lib:${LD_LIBRARY_PATH:-}"
+          } >> "$GITHUB_ENV"
+          echo "$(rocm-sdk path --bin)" >> "$GITHUB_PATH"
+          echo "$(pwd)/.rocm/bin" >> "$GITHUB_PATH"
+
+      - name: ccache key
+        id: cckey
+        run: echo "archs=$(echo "$GPU_TARGETS" | sha256sum | cut -c1-8)" >> "$GITHUB_OUTPUT"
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@d62db5f07c26379fc4b4e0916f098a92573c3b03  # v1.2.23
+        with:
+          key: sd-rocm-${{ env.ROCM_VERSION }}-${{ steps.cckey.outputs.archs }}-${{ needs.resolve.outputs.tag }}
+          restore-keys: |
+            sd-rocm-${{ env.ROCM_VERSION }}-${{ steps.cckey.outputs.archs }}
+          append-timestamp: false
+          variant: ccache
+          max-size: 2G
+          save: false
+
+      - name: Build sd-cli + sd-server (ROCm)
+        working-directory: src
+        run: |
+          set -euo pipefail
+          cmake -B build -G Ninja \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
+            -DCMAKE_HIP_FLAGS="-mllvm --amdgpu-unroll-threshold-local=600" \
+            -DSD_BUILD_EXAMPLES=ON \
+            -DSD_SERVER_BUILD_FRONTEND=OFF \
+            -DSD_WEBP=OFF -DSD_WEBM=OFF \
+            -DGGML_NATIVE=OFF \
+            -DSD_HIPBLAS=ON \
+            -DHIP_PLATFORM=amd \
+            -DGPU_TARGETS="$GPU_TARGETS" \
+            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_HIP_COMPILER_LAUNCHER=ccache
+          cmake --build build --config Release -j "$(nproc)" --target sd-cli sd-server
+          ccache --show-stats || true
+
+      - name: Stage the bundle with the ROCm runtime
+        run: |
+          set -euo pipefail
+          BIN="${GITHUB_WORKSPACE}/src/build/bin"
+          STAGE="${GITHUB_WORKSPACE}/rocm-bundle"
+          rm -rf "$STAGE" && mkdir -p "$STAGE/lib" "$STAGE/.kpack"
+          cp "$BIN/sd-cli" "$BIN/sd-server" "$STAGE/"
+          # ldd walks the closure and ggml is static here, so the two binaries are the whole list.
+          ldd "$STAGE/sd-cli" "$STAGE/sd-server" \
+            | awk -v r="$ROCM_PATH/" 'index($3, r) == 1 { print $3 }' | sort -u > "$RUNNER_TEMP/rocm-libs.txt"
+          if [ ! -s "$RUNNER_TEMP/rocm-libs.txt" ]; then
+            echo "ERROR: nothing resolved from $ROCM_PATH; is this a HIP build?" >&2
+            exit 1
+          fi
+          while read -r so; do cp -L "$so" "$STAGE/lib/"; done < "$RUNNER_TEMP/rocm-libs.txt"
+          # rocBLAS and hipBLASLt load kernels from <dir of the library>/<name>/library.
+          for t in rocblas hipblaslt; do
+            compgen -G "$STAGE/lib/lib$t.so*" > /dev/null || continue
+            mkdir -p "$STAGE/lib/$t"
+            cp -rL "$ROCM_PATH/lib/$t/library" "$STAGE/lib/$t/"
+          done
+          # kpack-split libraries name their GPU code in .rocm_kpack_ref as
+          # ../.kpack/<name>_@GFXARCH@.kpack, relative to the library, so lib/ and .kpack/ must
+          # stay siblings in the bundle.
+          for so in "$STAGE"/lib/*.so*; do
+            objcopy -O binary --only-section=.rocm_kpack_ref "$so" "$RUNNER_TEMP/kpack_ref.bin"
+            for name in $(strings "$RUNNER_TEMP/kpack_ref.bin" | grep -o '\.kpack/[A-Za-z0-9_]*_@GFXARCH@' \
+                | sed 's#^\.kpack/##; s#_@GFXARCH@$##' | sort -u); do
+              for gfx in ${GPU_TARGETS//;/ }; do
+                cp -L "$ROCM_PATH/.kpack/${name}_${gfx}.kpack" "$STAGE/.kpack/"
+              done
+            done
+          done
+          patchelf --set-rpath '$ORIGIN/lib' "$STAGE/sd-cli" "$STAGE/sd-server"
+          for f in "$STAGE"/lib/*.so*; do patchelf --set-rpath '$ORIGIN' "$f"; done
+          echo "bundled $(wc -l < "$RUNNER_TEMP/rocm-libs.txt") ROCm libraries, $(ls "$STAGE/.kpack" | wc -l) kpack archives"
+          du -sh "$STAGE"
+          # The proof the bundle stands alone: resolve with the wheel tree hidden.
+          if LD_LIBRARY_PATH= ldd "$STAGE/sd-cli" "$STAGE/sd-server" | grep -i 'not found'; then
+            echo "ERROR: unresolved libraries after bundling" >&2
+            exit 1
+          fi
+
+      - name: Package bundle
+        env:
+          BIN_DIR: ${{ github.workspace }}/rocm-bundle
+          KEEP_LAYOUT: "1"
+          OUT_DIR: ${{ github.workspace }}/dist
+          TAG: ${{ needs.resolve.outputs.tag }}
+          LABEL: Linux-Ubuntu-24.04-x86_64-rocm-7.14.0
+          COMMIT: ${{ needs.resolve.outputs.commit }}
+          SOURCE_REPO: ${{ github.repository }}
+          LICENSE_FILE: ${{ github.workspace }}/src/LICENSE
+        run: python3 tooling/scripts/unsloth/package_bundle.py
+
+      - name: Upload bundle
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: sd-${{ needs.resolve.outputs.tag }}-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.14.0
+          path: dist/sd-${{ needs.resolve.outputs.tag }}-bin-Linux-Ubuntu-24.04-x86_64-rocm-7.14.0.zip
+          if-no-files-found: error
+          retention-days: 7
+
+      - name: Evict stale ccache files
+        if: ${{ !cancelled() }}
+        continue-on-error: true
+        run: ccache --evict-older-than 14d
+
+      - name: Save ccache
+        if: ${{ always() }}
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
+        with:
+          path: ${{ github.workspace }}/.ccache
+          key: ccache-sd-rocm-${{ env.ROCM_VERSION }}-${{ steps.cckey.outputs.archs }}-${{ needs.resolve.outputs.tag }}-
+
   assemble:
     name: Assemble + publish
-    needs: [resolve, build-unix, build-windows, build-linux-cuda]
+    needs: [resolve, build-unix, build-windows, build-linux-cuda, build-windows-cuda, build-linux-rocm]
     # Consumed by `reclaim` to tell "these bundles are now release assets" from
     # "nothing will ever read these". Set only after draft=false lands.
     outputs:
@@ -727,6 +1265,7 @@ jobs:
 
       # Vulkan is required: the pinned upstream fallback lacks our H3 fixes.
       # Blocking publication on a Vulkan failure prevents promoting an incomplete fix.
+      # CUDA and ROCm are best effort and are not listed here.
       - name: Coverage gate (all 7 CPU/Apple/Vulkan assets present)
         run: |
           set -eu
@@ -776,7 +1315,7 @@ jobs:
           AHEAD='${{ needs.resolve.outputs.ahead }}'
           COMMIT='${{ needs.resolve.outputs.commit }}'
           REPO="$GITHUB_REPOSITORY"
-          NOTES="Automated Unsloth stable-diffusion.cpp prebuild (sd-cli + sd-server), built from [\`${COMMIT}\`](https://github.com/${REPO}/commit/${COMMIT}) in this repository. Includes CPU, Apple Metal, and Linux/Windows Vulkan builds, plus Linux CUDA when its build succeeds."
+          NOTES="Automated Unsloth stable-diffusion.cpp prebuild (sd-cli + sd-server), built from [\`${COMMIT}\`](https://github.com/${REPO}/commit/${COMMIT}) in this repository. Includes CPU, Apple Metal, and Linux/Windows Vulkan builds, plus Linux/Windows CUDA 12 and 13 and Linux ROCm when those builds succeed. The CUDA bundles carry no CUDA runtime: Studio supplies the one its llama.cpp build already uses."
           PINS='${{ needs.resolve.outputs.pins }}'
           if [ "${AHEAD:-0}" -gt 0 ]; then
             NOTES="${NOTES}"$'\n\n'"Not a stock build: the tree is ${AHEAD} commits past the ${UPSTREAM_TAG} release it descends from, which is what the \`-u\` suffix on the tag marks. \`sd-prebuilt-manifest.json\` records the commit."
diff --git a/.gitignore b/.gitignore
index 80599b3f81..b55ea0f81b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,10 @@ tests/
 .cache/
 *.swp
 *.bat
+!/start-server.bat
+/start-server.local.sh
+/start-server.local.bat
+/start-server.local.ps1
 *.bin
 *.exe
 *.gguf
diff --git a/AGENTS.md b/AGENTS.md
index 4953e2278b..a48040869d 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -83,6 +83,8 @@ documentation.
   and debugging.
 * `examples/common/` - shared example support code.
 * `examples/server/` - server application built on top of the library.
+  Before modifying built-in recipes, read and follow
+  [the built-in recipe policy](examples/server/recipes/README.md).
 * `examples/server/frontend/` - git submodule containing independent frontend
   code. Avoid modifying it unless explicitly requested.
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 134719ca47..d7bfda5999 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -225,6 +225,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
     "src/core/*.h"
     "src/core/*.cpp"
     "src/core/*.hpp"
+    "src/core/probe/*.h"
+    "src/core/probe/*.cpp"
     "src/extensions/*.h"
     "src/extensions/*.cpp"
     "src/extensions/*.hpp"
@@ -342,9 +344,17 @@ if (SD_CUDA)
     target_link_libraries(${SD_LIB} PUBLIC CUDA::cuda_driver)
     set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_CUDA)
 endif()
-target_include_directories(${SD_LIB} PUBLIC . src include)
-target_include_directories(${SD_LIB} PRIVATE src/core)
-target_include_directories(${SD_LIB} PUBLIC . thirdparty)
+if (SD_HIPBLAS)
+    find_package(hip REQUIRED CONFIG HINTS "$ENV{ROCM_PATH}" /opt/rocm)
+    target_link_libraries(${SD_LIB} PRIVATE hip::host)
+    set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_HIP)
+    if (GGML_HIP_MMQ_MFMA)
+        set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_HIP_MFMA)
+    endif()
+endif()
+target_include_directories(${SD_LIB} PUBLIC include)
+target_include_directories(${SD_LIB} PRIVATE src src/core)
+target_include_directories(${SD_LIB} PUBLIC thirdparty)
 target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
 
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d9975cf60d..ca64ad1981 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -12,6 +12,10 @@ If you want to update a third-party dependency, please open an issue first inste
 
 ## Pull Requests
 
+When contributing from a fork, use a fork under your personal GitHub account and enable **Allow edits from maintainers**. This lets maintainers make follow-up fixes directly on the PR branch.
+
+PRs from organization-owned forks are automatically closed when opened or reopened because GitHub does not support this maintainer-edit option for those forks. Submit the changes from a personal fork instead. See [GitHub's documentation](https://docs.github.com/en/pull-requests/how-tos/work-with-forks/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
+
 Keep each PR focused on one clear change. Large or overly complex PRs are harder to review and may not be merged.
 
 Do not include test code or test scripts in commits or PRs. Keep them local and report verification results in the PR description.
@@ -25,6 +29,21 @@ PRs should include:
 - Linked issue/PR context when applicable.
 - Screenshots or sample outputs for UI/visual behavior changes.
 
+## README UI Listings
+
+The [UIs list in the README](README.md#uis) only includes open-source projects that use `stable-diffusion.cpp` as a backend. Do not submit PRs to add closed-source UIs.
+
+## Server Recipes
+
+Before adding or changing a built-in server recipe, read and follow the
+[built-in recipe policy](examples/server/recipes/README.md). The policy governs
+native editing, community editing LoRAs, distinct task checkpoints, external
+adapters, native components and layer-count variants. Do not add recipes solely
+for style or acceleration LoRAs, separately stored native components, or added
+or removed model layers. Explain the task, component compatibility or capability
+difference that justifies a new recipe, and verify its required components,
+feature declarations and embedded API representation.
+
 ## Code Style
 
 Format code according to the repository style before submitting changes.
diff --git a/README.md b/README.md
index f184c51b0f..b4ab2f5ab2 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,8 @@ API and command-line option may change frequently.***
 
 ## 🔥Important News
 
+* **2026/10/16** 🚀 stable-diffusion.cpp adds **Day-0 support for MOSS-Image-Base / MOSS-Image-SFT**
+* **2026/10/16** 🚀 stable-diffusion.cpp adds **Day-0 support for FLUX.3 Image Klein / Klein Base**
 * **2026/09/20** 🚀 stable-diffusion.cpp adds **Day-0 support for Qwen-Image-2.1**
 * **2026/08/20** 🚀 stable-diffusion.cpp now supports **LTX-2.5**
 * **2026/08/04** 🚀 stable-diffusion.cpp adds **Day-1 support for MiniMax-H3**
@@ -44,6 +46,8 @@ API and command-line option may change frequently.***
     - [SD3/SD3.5](./docs/sd3.md)
     - [FLUX.1-dev/FLUX.1-schnell](./docs/flux.md)
     - [FLUX.2-dev/FLUX.2-klein](./docs/flux2.md)
+    - [FLUX.3 Image Klein / Klein Base](./docs/flux3.md)
+    - [MOSS-Image-Base / MOSS-Image-SFT](./docs/moss_image.md)
     - [Lens](./docs/lens.md)
     - [Chroma](./docs/chroma.md)
     - [Chroma1-Radiance](./docs/chroma_radiance.md)
@@ -52,6 +56,7 @@ API and command-line option may change frequently.***
     - [PiD](./docs/pid.md)
     - [LongCat Image](./docs/longcat_image.md)
     - [Z-Image](./docs/z_image.md)
+    - [Z-Image L2P](./docs/z_image_l2p.md)
     - [MiniT2I](./docs/minit2i.md)
     - [SenseNova U1.5](./docs/sensenova_u1.md)
     - [Ovis-Image](./docs/ovis_image.md)
@@ -64,6 +69,9 @@ API and command-line option may change frequently.***
     - [HiDream-O1-Image](./docs/hidream_o1_image.md)
     - [Ideogram4](./docs/ideogram4.md)
     - [LLaDA-Image](./docs/llada_image.md)
+    - [Ming-Image Design](./docs/ming_image.md)
+    - [PixArt](./docs/pixart.md)
+    - [Iris-3B](./docs/iris.md)
   - [Image Edit Models](./docs/edit.md)
     - [FLUX.1-Kontext-dev](./docs/kontext.md)
     - [Qwen Image Edit series](./docs/qwen_image_edit.md)
@@ -179,7 +187,6 @@ These projects wrap `stable-diffusion.cpp` for easier use in other languages/fra
 These projects use `stable-diffusion.cpp` as a backend for their image generation.
 
 - [GIMP Plugins](https://github.com/themanyone/gimp-plugins)
-- [Jellybox](https://jellybox.com)
 - [Stable Diffusion GUI](https://github.com/fszontagh/sd.cpp.gui.wx)
 - [Stable Diffusion CLI-GUI](https://github.com/piallai/stable-diffusion.cpp)
 - [Local Diffusion](https://github.com/rmatif/Local-Diffusion)
@@ -187,6 +194,7 @@ These projects use `stable-diffusion.cpp` as a backend for their image generatio
 - [LocalAI](https://github.com/mudler/LocalAI)
 - [Neural-Pixel](https://github.com/Luiz-Alcantara/Neural-Pixel)
 - [KoboldCpp](https://github.com/LostRuins/koboldcpp)
+- [Kokon](https://github.com/GabrieleLozupone/kokon)
 
 ## Contributors
 
diff --git a/assets/flux3/flux3-image-klein-base.png b/assets/flux3/flux3-image-klein-base.png
new file mode 100644
index 0000000000..411502269a
Binary files /dev/null and b/assets/flux3/flux3-image-klein-base.png differ
diff --git a/assets/flux3/flux3-image-klein-edit.png b/assets/flux3/flux3-image-klein-edit.png
new file mode 100644
index 0000000000..f392eb86d2
Binary files /dev/null and b/assets/flux3/flux3-image-klein-edit.png differ
diff --git a/assets/flux3/flux3-image-klein.png b/assets/flux3/flux3-image-klein.png
new file mode 100644
index 0000000000..7d45f34720
Binary files /dev/null and b/assets/flux3/flux3-image-klein.png differ
diff --git a/assets/iris/example.png b/assets/iris/example.png
new file mode 100644
index 0000000000..af3a4fd75b
Binary files /dev/null and b/assets/iris/example.png differ
diff --git a/assets/moss/sft.png b/assets/moss/sft.png
new file mode 100644
index 0000000000..ede42d1f78
Binary files /dev/null and b/assets/moss/sft.png differ
diff --git a/assets/qwen/qwen-image-2.1-alpha-in1.png b/assets/qwen/qwen-image-2.1-alpha-in1.png
new file mode 100644
index 0000000000..e65fcf0719
Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-in1.png differ
diff --git a/assets/qwen/qwen-image-2.1-alpha-out1.png b/assets/qwen/qwen-image-2.1-alpha-out1.png
new file mode 100644
index 0000000000..6841b7f787
Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-out1.png differ
diff --git a/assets/qwen/qwen-image-2.1-alpha-out2.png b/assets/qwen/qwen-image-2.1-alpha-out2.png
new file mode 100644
index 0000000000..8bef08a213
Binary files /dev/null and b/assets/qwen/qwen-image-2.1-alpha-out2.png differ
diff --git a/assets/wan/Wan2.2_A14B_vace_r2v.mp4 b/assets/wan/Wan2.2_A14B_vace_r2v.mp4
new file mode 100644
index 0000000000..caf19c2151
Binary files /dev/null and b/assets/wan/Wan2.2_A14B_vace_r2v.mp4 differ
diff --git a/cmake/ggml.cmake b/cmake/ggml.cmake
index 39d77eae35..d3e952dca4 100644
--- a/cmake/ggml.cmake
+++ b/cmake/ggml.cmake
@@ -21,6 +21,28 @@ endif()
 target_include_directories(${SD_LIB} PRIVATE "${sd_ggml_private_include}")
 set_property(TARGET ${SD_LIB} PROPERTY SD_GGML_PRIVATE_INCLUDE_DIR "${sd_ggml_private_include}")
 
+# The fused DiT ops come from scripts/unsloth/ggml-patches/0004; an unpatched ggml keeps the unfused H3 graph.
+set(sd_ggml_header "${sd_ggml_private_include}/../include/ggml.h")
+if(EXISTS "${sd_ggml_header}")
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${sd_ggml_header}")
+    file(STRINGS "${sd_ggml_header}" sd_ggml_h3_fused_ops REGEX "ggml_rope_pe_permute\\(")
+    file(STRINGS "${sd_ggml_header}" sd_ggml_h3_fused_qk_norm REGEX "ggml_rope_pe_permute_rms\\(")
+    file(STRINGS "${sd_ggml_header}" sd_ggml_rope_pe_interleaved REGEX "ggml_rope_pe_permute_set_interleaved\\(")
+endif()
+if(sd_ggml_h3_fused_ops)
+    target_compile_definitions(${SD_LIB} PUBLIC SD_GGML_H3_FUSED_OPS)
+else()
+    message(STATUS "ggml lacks the fused DiT ops patch: MiniMax-H3 uses the unfused graph")
+endif()
+# The fused q/k RMS norm comes from patch 0005.
+if(sd_ggml_h3_fused_qk_norm)
+    target_compile_definitions(${SD_LIB} PUBLIC SD_GGML_H3_FUSED_QK_NORM)
+endif()
+# Interleaved pairing for the fused RoPE op (patch 0007): the shared DiT attention (Rope::attention) uses it.
+if(sd_ggml_h3_fused_ops AND sd_ggml_rope_pe_interleaved)
+    target_compile_definitions(${SD_LIB} PUBLIC SD_GGML_FUSED_ROPE_ATTN)
+endif()
+
 if(SD_USE_UPSTREAM_GGML)
     target_compile_definitions(${SD_LIB} PUBLIC SD_USE_UPSTREAM_GGML)
     message(WARNING "Using upstream GGML: INT8 tensorwise/convrot is disabled and FP8 weights are converted to F16 at load time. Some operators may be unsupported and performance may be lower than with patched GGML.")
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 0ef3d56acc..3a80ae9bf8 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -40,4 +40,6 @@ RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]
diff --git a/docker/Dockerfile.cuda b/docker/Dockerfile.cuda
index 3e51bda60c..625dc60e70 100644
--- a/docker/Dockerfile.cuda
+++ b/docker/Dockerfile.cuda
@@ -57,4 +57,6 @@ RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]
diff --git a/docker/Dockerfile.musa b/docker/Dockerfile.musa
index 1c1d4d28c3..08d6f1d43d 100644
--- a/docker/Dockerfile.musa
+++ b/docker/Dockerfile.musa
@@ -1,7 +1,6 @@
-ARG MUSA_VERSION=rc4.2.0
-ARG UBUNTU_VERSION=22.04
+ARG MUSA_IMAGE=registry.mthreads.com/mcconline/inference/pytorch:2.9.1.post1-py3.10-musa5.2.0-mp31-devel-ubuntu22.04-amd64
 
-FROM mthreads/musa:${MUSA_VERSION}-devel-ubuntu${UBUNTU_VERSION}-amd64 as build
+FROM ${MUSA_IMAGE} AS build
 
 # sd-server embeds the web UI at build time, so the build image needs Node/pnpm.
 RUN apt-get update && apt-get install -y --no-install-recommends ccache cmake git ca-certificates curl gnupg && \
@@ -22,9 +21,8 @@ COPY . .
 
 RUN mkdir build && cd build && \
     cmake .. -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \
-        -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS} -fopenmp -I/usr/lib/llvm-14/lib/clang/14.0.0/include -L/usr/lib/llvm-14/lib" \
-        -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -fopenmp -I/usr/lib/llvm-14/lib/clang/14.0.0/include -L/usr/lib/llvm-14/lib" \
         -DSD_MUSA=ON \
+        -DMUSA_ARCHITECTURES=31 \
         -DSD_BUILD_SHARED_LIBS=ON \
         -DGGML_NATIVE=OFF \
         -DSD_BUILD_SHARED_GGML_LIB=ON \
@@ -35,11 +33,14 @@ RUN mkdir build && cd build && \
         -DCMAKE_BUILD_TYPE=Release && \
     cmake --build . --config Release
 
-FROM mthreads/musa:${MUSA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}-amd64 as runtime
+FROM ${MUSA_IMAGE} AS runtime
 
 COPY --from=build /sd.cpp/build/bin /sd.cpp/bin
 RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]
+CMD []
diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm
new file mode 100644
index 0000000000..fe8a9b15e1
--- /dev/null
+++ b/docker/Dockerfile.rocm
@@ -0,0 +1,50 @@
+FROM rocm/dev-ubuntu-24.04 AS build
+
+# sd-server embeds the web UI at build time, so the build image needs Node/pnpm.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential git cmake ninja-build ccache ca-certificates curl gnupg && \
+    mkdir -p /etc/apt/keyrings && \
+    curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key -o /tmp/nodesource-repo.gpg.key && \
+    gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg /tmp/nodesource-repo.gpg.key && \
+    rm /tmp/nodesource-repo.gpg.key && \
+    echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" > /etc/apt/sources.list.d/nodesource.list && \
+    apt-get update && apt-get install -y --no-install-recommends nodejs && \
+    npm install -g pnpm@10.15.1 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /sd.cpp
+
+# ROCm's clang is not on PATH in Docker RUN shells (profile scripts are not loaded).
+ENV PATH="/opt/rocm/llvm/bin:$PATH"
+
+COPY . .
+
+ARG AMDGPU_TARGETS=""
+
+RUN cmake . -B ./build -G Ninja \
+    -DCMAKE_C_COMPILER=clang \
+    -DCMAKE_CXX_COMPILER=clang++ \
+    -DSD_HIPBLAS=ON \
+    -DGGML_NATIVE=OFF \
+    -DAMDGPU_TARGETS="${AMDGPU_TARGETS}" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+RUN cmake --build ./build --config Release --parallel
+
+FROM rocm/dev-ubuntu-24.04 AS runtime
+
+# ROCm clang links against its own OpenMP runtime in /opt/rocm/llvm/lib,
+# and the ROCm runtime libs (hipblas, rocblas, amdhip64) live in /opt/rocm/lib.
+ENV LD_LIBRARY_PATH="/opt/rocm/llvm/lib:/opt/rocm/lib:$LD_LIBRARY_PATH"
+
+RUN apt-get update && \
+    apt-get install --yes --no-install-recommends libgomp1 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+COPY --from=build /sd.cpp/build/bin /sd.cpp/bin
+RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
+    printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
+    chmod +x /sd-cli /sd-server
+
+ENTRYPOINT [ "/sd-cli" ]
diff --git a/docker/Dockerfile.sycl b/docker/Dockerfile.sycl
index 9927ae7e58..9225e6eb2f 100644
--- a/docker/Dockerfile.sycl
+++ b/docker/Dockerfile.sycl
@@ -29,4 +29,6 @@ FROM intel/oneapi-basekit:${SYCL_VERSION}-devel-ubuntu24.04 AS runtime
 COPY --from=build /sd.cpp/build/bin/sd-cli /sd-cli
 COPY --from=build /sd.cpp/build/bin/sd-server /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]
\ No newline at end of file
diff --git a/docker/Dockerfile.vulkan b/docker/Dockerfile.vulkan
index 2c40c21a77..c7810a1fa5 100644
--- a/docker/Dockerfile.vulkan
+++ b/docker/Dockerfile.vulkan
@@ -41,4 +41,6 @@ RUN printf '#!/bin/sh\nexec /sd.cpp/bin/sd-cli "$@"\n' > /sd-cli && \
     printf '#!/bin/sh\nexec /sd.cpp/bin/sd-server "$@"\n' > /sd-server && \
     chmod +x /sd-cli /sd-server
 
+WORKDIR /sd.cpp
+
 ENTRYPOINT [ "/sd-cli" ]
diff --git a/docs/anima.md b/docs/anima.md
index b852358fdd..18b94952b6 100644
--- a/docs/anima.md
+++ b/docs/anima.md
@@ -15,7 +15,7 @@
 ## Examples
 
 ```sh
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\anima-preview.safetensors --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\qwen_3_06b_base.safetensors  -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\anima-preview.safetensors --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\qwen_3_06b_base.safetensors  -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu
 ```
 
 <img alt="anima image example" src="../assets/anima/example.png" />
diff --git a/docs/animatediff.md b/docs/animatediff.md
index 2e923b0bf3..80c46ae64f 100644
--- a/docs/animatediff.md
+++ b/docs/animatediff.md
@@ -51,7 +51,7 @@ SD 1.5 defaults (euler_a, low CFG) produces noise-like output.
 .\bin\Release\sd-cli.exe -M vid_gen \
     --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
     --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
-    --offload-to-cpu --diffusion-fa \
+    --offload-to-cpu \
     -p "a red apple on a wooden table" \
     --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
     -H 512 -W 512 --video-frames 8 --fps 8 --steps 20 -s 42 \
@@ -66,7 +66,7 @@ use layer streaming:
 .\bin\Release\sd-cli.exe -M vid_gen \
     --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
     --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
-    --offload-to-cpu --diffusion-fa \
+    --offload-to-cpu \
     -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
     --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
     -H 384 -W 384 --video-frames 16 --fps 8 --steps 20 -s 42 \
@@ -79,7 +79,7 @@ Low-VRAM streaming (verified with a 2 GiB cap on RTX 3060):
 .\bin\Release\sd-cli.exe -M vid_gen \
     --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
     --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
-    --max-vram 2.0 --diffusion-fa \
+    --max-vram 2.0 \
     -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
     --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
     -H 384 -W 384 --video-frames 8 --fps 8 --steps 20 -s 42 \
diff --git a/docs/backend.md b/docs/backend.md
index 846aafb858..057d3f20ef 100644
--- a/docs/backend.md
+++ b/docs/backend.md
@@ -65,8 +65,12 @@ sd-cli -m model.safetensors -p "a cat" --backend "diffusion=cuda0&cuda1"
 ```
 
 The module's transformer blocks are then distributed across the listed devices
-in contiguous ranges sized proportionally to each device's free memory (minus a
-compute-buffer headroom of about 2 GiB per device), and the
+in contiguous ranges sized proportionally to each device's weight capacity: its
+free memory or its `--max-vram` budget, whichever is lower, minus a
+compute-buffer headroom of about 2 GiB. Every device therefore keeps the same
+fraction of its budget for the compute buffers of the blocks it runs. If the
+proportional ranges do not fit (very uneven block sizes), each device is filled
+in turn up to its capacity instead. The
 module's graphs are executed with a `ggml_backend_sched` that runs each block
 on the device holding its weights, copying the residual stream at the range
 boundaries. The first device in the list is the module's main device: it also
@@ -156,8 +160,14 @@ the runner's graph-cut capacity checks.
 
 Runtime capacity checks also leave 512 MiB of currently free device memory for
 backend scratch buffers and pipelines, including with explicit backend assignments.
-They cap stale free-memory reports by the device's total memory minus tracked
-resident allocations and reject reports that exceed the device's total memory.
+They cap free-memory reports by the device's total memory minus tracked
+resident allocations. Vulkan reports exceeding total memory are rejected because
+its heap-budget subtraction can underflow. Other backends use the cap instead of
+treating such reports as zero free memory. Failed checks log the reported free and
+total memory alongside tracked weight and runtime allocations.
+With `--mmap`, device-backed mappings count toward these budgets at their full
+mapped-file size, once per device buffer even when multiple parameter blocks
+share it. Mappings retained in the loader cache continue to count.
 
 Components are considered in `diffusion`, `te`, `vae` order so that repeatedly
 used diffusion weights have priority. Each component's weights use the first
@@ -209,6 +219,123 @@ If a VAE decode fails, decoding retries with spatial tiling even when `--auto-fi
 is off; supported video decoders try temporal tiling first and can then add
 spatial tiling. Spatial retries use half-size tiles along each latent dimension.
 
+## Direct convolution (`--conv-direct on|off|auto`)
+
+`--conv-direct` defaults to `auto` and accepts module assignments for
+`diffusion`, `vae` (including preview VAE), and `controlnet`, using the same
+aliases and precedence rules as `--flash-attn`:
+
+```shell
+sd-cli -m model.safetensors -p "a cat" --conv-direct auto,vae=off
+sd-cli -m model.safetensors -p "a cat" --conv-direct off,diffusion=on
+```
+
+- `on` requests direct 2D and 3D convolution.
+- `off` uses the existing im2col-based convolution paths.
+- `auto` first checks support for a representative F16-weight/F32-input direct
+  convolution graph. Vulkan then benchmarks direct convolution against im2col
+  using two small shapes per dimension, with warmup, output validation, and five
+  interleaved measurements per shape. Direct convolution must improve the sum of
+  median times by at least 5%; a failed probe defaults to `off`.
+  Other backends retain their implicit GEMM checks without benchmarking:
+  CUDA 2D requires a compiled Turing-or-newer MMA architecture; HIP 2D requires
+  the supported CDNA MFMA or RDNA WMMA architectures described below for flash
+  attention. MUSA 2D has a SIMT implementation. CUDA, HIP, and MUSA 3D also have
+  a SIMT implementation, so they do not require MMA. Other backends default to
+  `off`. Results and decision logs are cached per compute backend and dimension
+  in the context, as with flash attention. Each dimension is selected independently,
+  and all compute devices in a module must qualify. Parameter-storage devices do
+  not affect this choice.
+
+These are backend-level defaults. GGML still chooses the actual kernel based on
+weight type, shape, and hardware; enabling direct convolution does not guarantee
+that every operation uses implicit GEMM. The CUDA-family implicit GEMM paths
+require F16 weights and can use ordinary direct kernels for other cases.
+This option does not change the separately created image upscaler context.
+
+Repeating `--conv-direct` replaces the previous assignment string. Existing
+`--diffusion-conv-direct` (diffusion and ControlNet) and `--vae-conv-direct`
+flags are deprecated and emit a warning, but remain supported when
+`--conv-direct` is omitted. Use `--conv-direct auto,diffusion=on,controlnet=on`
+and `--conv-direct auto,vae=on`, respectively. An explicit
+`--conv-direct` takes precedence over both legacy flags regardless of order.
+Library callers can set the appended `sd_ctx_params_t::conv_direct_mode` field
+to the same string; null or empty selects `auto` while honoring legacy true
+flags. The context copies the string.
+
+## Flash attention (`--flash-attn on|off|auto`)
+
+`--flash-attn` defaults to `auto`. Selection happens while creating the model
+context, before inference, and is cached once per compute backend in that context.
+Parameter-storage backends are not benchmarked.
+
+The option also accepts module assignments, using the same module aliases as
+`--backend`:
+
+```shell
+sd-cli -m model.safetensors -p "a cat" --flash-attn auto,te=off,vae=on
+sd-cli -m model.safetensors -p "a cat" --flash-attn off,diffusion=auto
+```
+
+Supported modules are `diffusion`, `te`, `clip_vision`, `vae`, `controlnet`, and
+`audio_encoder`. Modules without an assignment use the default (`auto` if no
+default is supplied). A bare mode or `all=`, `default=`, or `*=` sets the default;
+module-specific settings take precedence regardless of their position. Within
+one assignment string, the last setting for the same module or default wins.
+Repeating `--flash-attn` replaces the entire previous string.
+
+Only modules set to `auto` need backend probing. An explicit `on` or `off` for
+one module does not change another module's decision, even when they share a
+backend. Modules without configurable attention are rejected instead of silently
+ignoring their settings.
+
+- `on` requests flash attention for all models. Unsupported attention operations
+  still fall back to ordinary attention.
+- `off` disables flash attention, including the flash-attention fallback from
+  SageAttention. SageAttention itself remains controlled by `--sage-attn`.
+- `auto` enables CPU and supported CUDA MMA configurations without timing a
+  benchmark. CUDA uses both the device's capability and the compiled GGML
+  architectures. HIP also skips timing on known CDNA MFMA architectures
+  (`gfx908`, `gfx90a`, `gfx940`/`gfx941`/`gfx942`, `gfx950`) and RDNA 3/3.5/4
+  WMMA architectures (`gfx11xx`/`gfx12xx`). The CDNA shortcut requires
+  `GGML_HIP_MMQ_MFMA` to be enabled at build time. All shortcuts still check
+  support for the FA probe graph. Vulkan devices whose description identifies
+  NVIDIA RTX hardware (GeForce RTX, TITAN RTX, Quadro RTX and professional RTX)
+  also skip timing. This is a device-name heuristic and needs no CUDA runtime.
+  Unsupported graphs are disabled; other configurations, including older or
+  unknown HIP architectures and other Vulkan devices, are benchmarked.
+
+The probe compares ordinary and flash attention through the same attention
+builder used by models, including layout conversion and F16 conversion. It uses
+two small self-attention graphs (512 tokens with head size 64, and 1024 tokens
+with head size 128, both with four heads), checks their outputs, warms up each
+path, then alternates five synchronized measurements. Flash attention is enabled
+when the sum of its median times is at least 5% lower. These representative
+shapes are a startup heuristic, not a guarantee for every model or resolution.
+Allocation/execution failures and inconclusive probes select `off`. Timing stops
+starting new measurement pairs after one second of measured computation;
+backend initialization, warmup and an in-flight pair can take longer.
+
+Different modules can select different results. A multi-device model enables
+flash attention only if all of its compute devices select `on`, preserving the
+model's existing single boolean setting. This conservative rule also covers
+row-split configurations, which can fall back to layer split during initialization.
+
+`--fa` and `--diffusion-fa` are deprecated and emit a warning when used.
+Use `--flash-attn on` and `--flash-attn off,diffusion=on`, respectively.
+Both aliases write the same mode string as `--flash-attn`; the last occurrence
+of any of these options wins. Library callers
+can set `sd_ctx_params_t::flash_attn_mode` to the same string, including
+`"on"`, `"off"`, `"auto"`, or `"auto,te=off,vae=on"`.
+A null or empty string uses automatic selection unless a
+legacy `flash_attn` or `diffusion_flash_attn` boolean is set. The context copies
+the string; no new enum is required.
+
+```shell
+sd-cli -m model.safetensors -p "a cat" --flash-attn auto
+sd-cli -m model.safetensors -p "a cat" --flash-attn off
+```
+
 ## Modules
 
 | Module | Purpose | Accepted names |
diff --git a/docs/backend_probe.md b/docs/backend_probe.md
new file mode 100644
index 0000000000..c420326d4c
--- /dev/null
+++ b/docs/backend_probe.md
@@ -0,0 +1,50 @@
+# Adding backend performance probes
+
+`src/core/probe/backend_probe.h` provides the shared measurement machinery. Operator
+policy stays separate: `src/core/probe/flash_attn.cpp`, for example, defines attention
+shapes and its CPU/CUDA/HIP/Vulkan shortcuts, then calls `sd_run_backend_probe` for unknown
+configurations. `src/core/probe/conv_direct.cpp` uses metadata-only probe graphs
+to check direct 2D/3D convolution support, then enables known implicit GEMM
+paths on non-Vulkan backends. Vulkan compares direct convolution with im2col
+using the shared measurement machinery.
+
+For each representative shape, create a `BackendProbeCase`:
+
+- `build(graph, candidate)` creates inputs and either the reference or candidate
+  graph in `graph.ctx`, registers input tensors in `graph.inputs`, and assigns
+  `graph.output`. Leave the output null if the requested implementation is
+  unavailable; do not silently benchmark its fallback as the candidate.
+- By default, inputs receive deterministic F32 data, with the same seed for both
+  variants. Supply `initialize` for F16 convolution weights, integer inputs, or
+  operator-specific values. It runs after allocation, outside timed execution.
+  Both variants must receive equivalent logical inputs.
+- Default output validation requires matching shapes and contiguous F32 outputs,
+  checks finiteness, and applies absolute and relative tolerances. Supply
+  `compare` for other output types or validation rules.
+
+Graphs must be safe to execute repeatedly without modifying their inputs.
+`BackendProbeGraph` owns its GGML context and buffer, so all return paths release
+probe resources. Constructors allocate metadata only; `supported()` can also be
+used before a known-architecture shortcut, without allocating device tensors.
+
+`BackendProbeOptions` controls graph metadata capacity, warmup count, measurement
+count, measured-time budget, minimum free device memory, comparison tolerances,
+and minimum improvement. Warmup and allocation are excluded from timing. The
+runner alternates the two variants, synchronizes the backend around execution,
+and compares sums of per-case medians. The time budget is checked before each
+measurement pair; it cannot interrupt an in-flight backend operation.
+
+Only use `candidate_faster` when `completed` is true. On incomplete probes,
+`error` reports why comparison failed and the timing totals may be partial.
+The caller handles logging, known architecture rules, fallback policy, and
+context-lifetime caching. Keep caches separate by operation (or key a shared
+cache by both operation and backend); an attention result must not select a
+convolution implementation. If shapes or settings vary within one context,
+include them in the corresponding cache key as well.
+
+Convolution results are cached separately for 2D and 3D, once per compute backend
+in the context, just like attention results. Each probe logs its backend, decision,
+and reason; mixed-device modules log when their compute devices disagree.
+
+Module assignment parsing remains in `src/core/ggml_extend_backend.cpp`, next
+to backend assignment parsing. The probe directory does not parse CLI/API options.
diff --git a/docs/boogu_image.md b/docs/boogu_image.md
index 2744aae57a..c5ad97dd85 100644
--- a/docs/boogu_image.md
+++ b/docs/boogu_image.md
@@ -17,7 +17,7 @@ Boogu Image uses a Boogu diffusion transformer, the FLUX VAE, and Qwen3-VL as th
 ### Boogu Image Base
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_base_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\ae.sft -p "a lovely cat" --diffusion-fa -v --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_base_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\ae.sft -p "a lovely cat" -v --offload-to-cpu
 ```
 
 <img width="256" alt="Boogu Image Base example" src="../assets/boogu/example.png" />
@@ -25,7 +25,7 @@ Boogu Image uses a Boogu diffusion transformer, the FLUX VAE, and Qwen3-VL as th
 ### Boogu Image Edit
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_edit_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\mmproj-Qwen3VL-8B-Instruct-F16.gguf --vae ..\models\vae\ae.sft --diffusion-fa -v --offload-to-cpu -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'boogu.cpp'"
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_edit_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\mmproj-Qwen3VL-8B-Instruct-F16.gguf --vae ..\models\vae\ae.sft -v --offload-to-cpu -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'boogu.cpp'"
 ```
 
 <img width="256" alt="Boogu Image Edit example" src="../assets/boogu/edit_example.png" />
\ No newline at end of file
diff --git a/docs/build.md b/docs/build.md
index 9daaa219b4..6ec0e40f0a 100644
--- a/docs/build.md
+++ b/docs/build.md
@@ -119,14 +119,16 @@ cmake --build . --config Release
 
 ## Build with MUSA
 
-This provides GPU acceleration using Moore Threads GPU. Make sure to have the MUSA toolkit installed.
+This provides GPU acceleration using Moore Threads GPUs. The configuration below follows upstream validation with MUSA SDK 5.2.0 and MTT S5000 (PH1, `mp_31`). Install that toolkit before building. Older SDKs such as 4.2 may lack the CUB headers required by the current backend.
 
 ```shell
 mkdir build && cd build
-cmake .. -DCMAKE_C_COMPILER=/usr/local/musa/bin/clang -DCMAKE_CXX_COMPILER=/usr/local/musa/bin/clang++ -DSD_MUSA=ON -DCMAKE_BUILD_TYPE=Release
+cmake .. -DCMAKE_C_COMPILER=/usr/local/musa/bin/clang -DCMAKE_CXX_COMPILER=/usr/local/musa/bin/clang++ -DSD_MUSA=ON -DMUSA_ARCHITECTURES=31 -DCMAKE_BUILD_TYPE=Release
 cmake --build . --config Release
 ```
 
+The MUSA Docker image uses the same SDK 5.2.0 development image as upstream for both build and runtime stages, and targets `mp_31` only. It includes the development tools and PyTorch from that base image, so it is larger than a minimal runtime image. Other GPU architectures require separate validation.
+
 ## Build with Metal
 
 Using Metal makes the computation run on the GPU. Currently, there are some issues with Metal when performing operations on very large matrices, making it highly inefficient at the moment. Performance improvements are expected in the near future.
diff --git a/docs/caching.md b/docs/caching.md
index 01f0197443..f69cfbeca8 100644
--- a/docs/caching.md
+++ b/docs/caching.md
@@ -2,6 +2,16 @@
 
 Caching methods accelerate diffusion inference by reusing intermediate computations when changes between steps are small.
 
+### Conditioning Cache
+
+Conditioning results are cached per model context using an LRU cache. The default
+capacity is **0 (disabled) for `sd-cli`** and **4 entries for `sd-server` and the C
+API**. Set `--conditioning-cache-size N` to change the limit; `0` disables caching.
+For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 4`
+enables the cache in the CLI. The C API option is
+`sd_ctx_params_t::conditioning_cache_size`, initialized by `sd_ctx_params_init()`.
+This cache is independent of the diffusion-step `--cache-mode` options below.
+
 ### Cache Modes
 
 | Mode | Target | Description |
diff --git a/docs/docker.md b/docs/docker.md
index 5bacdd2d4b..28e9ab5dfa 100644
--- a/docs/docker.md
+++ b/docs/docker.md
@@ -24,12 +24,25 @@ docker build -f docker/Dockerfile -t sd .
 
 ## Building variants using Docker
 
-Vulkan:
+### Vulkan:
 
 ```shell
 docker build -f docker/Dockerfile.vulkan -t sd .
 ```
 
+### ROCm (AMD GPU):
+
+Defining one or multiple targets with `--build-arg AMDGPU_TARGETS` is strongly recommended.
+Building the image will otherwise likely fail or build for a target that does not fit your
+hardware.
+
+```shell
+# Build for one target
+docker build -f docker/Dockerfile.rocm --build-arg "AMDGPU_TARGETS=gfx1100" -t sd .
+# Build for multiple targets
+docker build -f docker/Dockerfile.rocm --build-arg "AMDGPU_TARGETS=gfx1030;gfx1100" -t sd .
+```
+
 ## Run locally built image's CLI
 
 ```shell
@@ -37,3 +50,21 @@ docker run --rm -v /path/to/models:/models -v /path/to/output/:/output sd [args.
 # For example
 # docker run --rm -v ./models:/models -v ./build:/output sd -m /models/sd-v1-4.ckpt -p "a lovely cat" -v -o /output/output.png
 ```
+
+### ROCm (AMD GPU)
+
+The user that executes this command needs to be in the `video` group and have read and write
+access to `/dev/dri` and `/dev/kfd`.
+
+`--device` and `--group-add` allow the container access to the GPU.
+
+```shell
+docker run --rm --device=/dev/dri --device=/dev/kfd --group-add video \
+  -v /path/to/models:/models -v /path/to/output/:/output \
+  sd [args...]
+# For example
+# docker run --rm --device=/dev/dri --device=/dev/kfd --group-add video \
+#   -v ./models:/models -v ./build:/output \
+#   sd -m /models/sd-v1-4.ckpt -v -o /output/output.png \
+#   -p "a lovely cat"
+```
\ No newline at end of file
diff --git a/docs/edit.md b/docs/edit.md
index 9f8a9c3f3b..62ecff2af5 100644
--- a/docs/edit.md
+++ b/docs/edit.md
@@ -14,6 +14,7 @@ Depending on the architecture, different models handle reference images differen
 | [**Qwen Image Edit**](./qwen_image_edit.md) | `qwen` |
 | **Qwen Image LAYERED** | `qwen_layered` |
 | [**Flux.2 [Dev] / Flux.2 [Klein]**](./flux2.md) | `flux2` |
+| [**FLUX.3 Image Klein / Klein Base**](./flux3.md) | `flux2` |
 | [**Boogu Image Edit**](./boogu_image.md) | `z_image_omni` |
 | **Krea2 (Community Edit LoRAs)** | `krea2_ostris_edit` |
 | [**Mage-Flow-Edit**](./mage_flow.md#image-editing) | `mage_flow` |
@@ -26,6 +27,9 @@ Stable-diffusion.spp also supports basic Unet-based editing models like instruct
 
 ## Configuring Reference Modes (`--ref-image-args`)
 
+For a one-time input transform before reference presets and model processing,
+including cropping, padding, and resizing algorithms, see [Image preprocessing](./image_preprocessing.md).
+
 Different DiT-based editing models require different configurations to process reference images correctly (e.g., whether to use a Vision Language Model (VLM) encoder or pass VAE-encoded images directly to the DiT).
 
 To simplify this, we provide **Presets**. By default, the system automatically selects the best preset based on the model architecture. However, you can override this using the `--ref-image-args` argument.
diff --git a/docs/ernie_image.md b/docs/ernie_image.md
index 99f942211e..3b1fc124cb 100644
--- a/docs/ernie_image.md
+++ b/docs/ernie_image.md
@@ -21,7 +21,7 @@ You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — o
 ### ERNIE-Image-Turbo
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu
 ```
 
 <img width="256" alt="ERNIE-Image Turbo example" src="../assets/ernie_image/turbo_example.png" />
@@ -29,7 +29,7 @@ You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — o
 ### ERNIE-Image
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu
 ```
 
 <img width="256" alt="ERNIE-Image example" src="../assets/ernie_image/example.png" />
diff --git a/docs/esrgan.md b/docs/esrgan.md
index 39a9760502..4068ef11c7 100644
--- a/docs/esrgan.md
+++ b/docs/esrgan.md
@@ -2,6 +2,10 @@
 
 You can use ESRGAN—such as the model [RealESRGAN_x4plus_anime_6B.pth](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.2.4/RealESRGAN_x4plus_anime_6B.pth)—to upscale the generated images and improve their overall resolution and clarity.
 
+MoSR models with the PixelShuffle upsampler, such as [4xNomos2_hq_mosr.pth](https://github.com/Phhofm/models/releases/download/4xNomos2_hq_mosr/4xNomos2_hq_mosr.pth), are also supported. The architecture is detected from the weights, so they are used the same way as ESRGAN models.
+
+RGBA images, including Qwen Image 2.1 output, keep their alpha channel during model upscaling and hires fix. The upscaler processes the RGB channels; the alpha channel is resized with bilinear interpolation and recombined with the upscaled image.
+
 - Specify the model path using the `--upscale-model PATH` parameter. example:
 
 ```bash
diff --git a/docs/flux2.md b/docs/flux2.md
index d3b2eff1f0..53fc8be5cf 100644
--- a/docs/flux2.md
+++ b/docs/flux2.md
@@ -16,7 +16,7 @@
 ### Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu
 ```
 
 <img alt="flux2 example" src="../assets/flux2/example.png" />
@@ -42,19 +42,19 @@
 ### Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu
 ```
 
 <img alt="flux2-klein-4b" src="../assets/flux2/flux2-klein-4b.png" />
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 4
 ```
 
 <img alt="flux2-klein-4b-edit" src="../assets/flux2/flux2-klein-4b-edit.png" />
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu
 ```
 
 <img alt="flux2-klein-base-4b" src="../assets/flux2/flux2-klein-base-4b.png" />
@@ -78,19 +78,19 @@
 ### Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu
 ```
 
 <img alt="flux2-klein-9b" src="../assets/flux2/flux2-klein-9b.png" />
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 4
 ```
 
 <img alt="flux2-klein-9b-edit" src="../assets/flux2/flux2-klein-9b-edit.png" />
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu
 ```
 
 <img alt="flux2-klein-base-9b" src="../assets/flux2/flux2-klein-base-9b.png" />
\ No newline at end of file
diff --git a/docs/flux3.md b/docs/flux3.md
new file mode 100644
index 0000000000..8bc0b2ec29
--- /dev/null
+++ b/docs/flux3.md
@@ -0,0 +1,50 @@
+# How to Use
+
+FLUX.3 Image Klein uses a FLUX.3 diffusion transformer, the FLUX.2 VAE, and Qwen3-VL 4B as the LLM text encoder.
+
+## Download weights
+
+- Download FLUX.3 Image Klein
+    - safetensors: https://huggingface.co/black-forest-labs/flux-3-image-klein/tree/main
+    - gguf: https://huggingface.co/leejet/flux-3-image-klein-GGUF/tree/main
+- Download FLUX.3 Image Klein Base
+    - safetensors: https://huggingface.co/black-forest-labs/flux-3-image-klein-base/tree/main
+    - gguf: https://huggingface.co/leejet/flux-3-image-klein-base-GGUF/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/black-forest-labs/FLUX.2-dev/tree/main
+    - The `ae.safetensors` included with FLUX.3 can also be used.
+- Download Qwen3-VL 4B
+    - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders
+    - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main
+
+Use the root `model.safetensors` for the diffusion model.
+
+## Examples
+
+### FLUX.3 Image Klein
+
+The distilled model uses 16 steps and CFG 1.0.
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-3-image-klein.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p "a lovely cat holding a sign says 'flux3.cpp'" --cfg-scale 1.0 --steps 16 --sampling-method euler -v --offload-to-cpu
+```
+
+<img src="../assets/flux3/flux3-image-klein.png" />
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-3-image-klein.safetensors --vae ..\models\vae\flux2_ae.safetensors --llm  ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'flux3-image-klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v  --offload-to-cpu --steps 16
+```
+
+<img src="../assets/flux3/flux3-image-klein-edit.png" />
+
+Repeat `-r` to use multiple reference images, up to 10. See [Image Editing](./edit.md) for reference-image preprocessing options.
+
+### FLUX.3 Image Klein Base
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\flux-3-image-klein-base.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p "a lovely cat holding a sign says 'flux3.cpp'" --cfg-scale 3.5 --steps 20 --sampling-method euler -v --offload-to-cpu
+```
+
+<img src="../assets/flux3/flux3-image-klein-base.png" />
+
+The default discrete scheduler uses the upstream time shift of 6.93 for both models. Use `--flow-shift` to override it.
diff --git a/docs/hunyuan_video.md b/docs/hunyuan_video.md
index 2b05e0f718..fadb884ef4 100644
--- a/docs/hunyuan_video.md
+++ b/docs/hunyuan_video.md
@@ -18,7 +18,7 @@ and ByT5 Small GlyphXL for glyph-aware text conditioning.
 ## Text-to-video example
 
 ```shell
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\hunyuanvideo1.5_720p_t2v_fp16.safetensors --vae ..\models\vae\hunyuanvideo15_vae_fp16.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --t5xxl ..\models\text_encoders\byt5_small_glyphxl_fp16.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --diffusion-fa --video-frames 33 --vae-tiling
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\hunyuanvideo1.5_720p_t2v_fp16.safetensors --vae ..\models\vae\hunyuanvideo15_vae_fp16.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --t5xxl ..\models\text_encoders\byt5_small_glyphxl_fp16.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --video-frames 33 --vae-tiling
 ```
 
 <video src=../assets/hunyuan_video/hy1.5_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
\ No newline at end of file
diff --git a/docs/ideogram4.md b/docs/ideogram4.md
index 003a02d594..f0f1dd2500 100644
--- a/docs/ideogram4.md
+++ b/docs/ideogram4.md
@@ -49,7 +49,7 @@ If you want lower VRAM usage, you can change the quantization from q8_0 to a low
 ## Examples
 
 ```sh
-.\bin\Release\sd-cli.exe --diffusion-model ideogram4-Q8_0.gguf --uncond-diffusion-model ideogram4_uncond-Q8_0.gguf --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p '{"high_level_description":"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.","style_description":{"aesthetics":"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished","lighting":"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background","photo":"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field","medium":"mixed media fashion photography and premium editorial graphic design","color_palette":["#F4EFE7","#111111","#D8B56D","#B73A3A","#FFFFFF","#8A7A6A"]},"compositional_deconstruction":{"canvas":"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.","background":"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.","layout":"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.","elements":[{"type":"text","desc":"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text."},{"type":"obj","desc":"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated."},{"type":"obj","desc":"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude."},{"type":"obj","desc":"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate."},{"type":"obj","desc":"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail."},{"type":"obj","desc":"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing."},{"type":"text","desc":"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right."},{"type":"obj","desc":"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations."},{"type":"text","desc":"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text."}]}}'  --diffusion-fa -v --offload-to-cpu -H 1024 -W 1024
+.\bin\Release\sd-cli.exe --diffusion-model ideogram4-Q8_0.gguf --uncond-diffusion-model ideogram4_uncond-Q8_0.gguf --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors -p '{"high_level_description":"A square 1024 x 1024 luxury fashion magazine cover featuring exactly one short chubby fluffy cat as the main model. The cat sits on a soft ivory studio floor, facing the viewer with a stylish calm expression, wearing tiny black sunglasses, a red silk scarf, and a small gold collar charm. In front of the cat on the floor is a wide horizontal luxury nameplate that clearly reads ideogram4.cpp. The whole design feels premium, fashionable, clean, and editorial.","style_description":{"aesthetics":"luxury fashion magazine cover, high-end pet couture campaign, minimalist editorial design, elegant studio photography, soft paper texture, refined typography, fashionable and polished","lighting":"Soft diffused studio lighting, gentle spotlight on the cat, subtle floor shadow, warm ivory highlights, clean separation between subject and background","photo":"high-resolution fashion editorial photography look, front-facing cat portrait, crisp fur details, glossy sunglasses, clear readable nameplate text, shallow depth of field","medium":"mixed media fashion photography and premium editorial graphic design","color_palette":["#F4EFE7","#111111","#D8B56D","#B73A3A","#FFFFFF","#8A7A6A"]},"compositional_deconstruction":{"canvas":"Square 1024 x 1024 canvas with a normal upright orientation. Do not rotate the poster or any text. Use a clean fashion magazine cover layout.","background":"Warm ivory studio backdrop with subtle paper grain, a soft spotlight gradient, faint floor shadow, and a few minimal gold editorial lines. The background is spacious, premium, and uncluttered.","layout":"Top center has a small elegant headline. Center area features one cat as the main fashion model. Lower foreground has a wide horizontal luxury nameplate placed on the floor in front of the cat. Bottom center has a small footer. All text is horizontal, upright, and readable left to right.","elements":[{"type":"text","desc":"Top center headline reading LOOK WHAT I FOUND in a refined high-fashion serif font. The headline is horizontal, centered, elegant, and secondary to the nameplate text."},{"type":"obj","desc":"Exactly one short chubby fluffy cat sitting in the center like a luxury fashion model. The cat has a large round head, compact body, short legs, soft detailed fur, expressive eyes, and a calm confident pose. The cat is cute and rounded, not tall, not stretched, not duplicated."},{"type":"obj","desc":"Tiny glossy black sunglasses worn naturally by the cat, slightly oversized but still showing the cat face clearly. The sunglasses add a chic fashion-editorial attitude."},{"type":"obj","desc":"A red silk scarf tied neatly around the cat neck, with soft folds and a couture feeling. The scarf must not cover the cat face or the nameplate."},{"type":"obj","desc":"A small gold collar charm or fashion accessory under the scarf, subtle and premium, adding a luxury campaign detail."},{"type":"obj","desc":"In the lower foreground, place a wide horizontal luxury nameplate on the floor in front of the cat. The nameplate is low, flat, landscape-oriented, much wider than tall, like a fashion show seat card or premium display plaque. It is centered, front-facing, level, and fully visible. It must not become vertical, tall, standing, rotated, or side-facing."},{"type":"text","desc":"Print the exact text ideogram4.cpp only on the wide horizontal nameplate. Use clean bold black lettering, perfectly spelled, lowercase, with the number 4 and .cpp extension. The text must fit completely inside the nameplate, stay horizontal, and be readable from left to right."},{"type":"obj","desc":"Add sparse premium editorial accents around the edges: thin gold lines, small code brackets, tiny cursor marks, subtle dots, and minimal geometric details. No extra cats, no stickers, no animal faces, no busy decorations."},{"type":"text","desc":"Bottom center footer reading tiny paws, big compile energy in a small refined monospace or editorial font. The footer is horizontal, centered, understated, and much smaller than the nameplate text."}]}}' -v --offload-to-cpu -H 1024 -W 1024
 ```
 
 <img alt="ideogram4 image example" src="../assets/ideogram4/example.png" />
diff --git a/docs/image_preprocessing.md b/docs/image_preprocessing.md
new file mode 100644
index 0000000000..719781750e
--- /dev/null
+++ b/docs/image_preprocessing.md
@@ -0,0 +1,173 @@
+# Image preprocessing
+
+Use `--image-preprocess` to transform each image input once, before generation:
+
+```sh
+sd-cli ... \
+  --image-preprocess "target=init,mode=crop-resize,filter=lanczos,antialias=true" \
+  --image-preprocess "target=mask,filter=nearest-exact" \
+  --image-preprocess "target=ref,index=0,mode=fit-pad,width=768,height=768,filter=bicubic"
+```
+
+CLI and server image loaders decode at the original resolution. The generation
+entry point merges input defaults with user rules and prepares one transformed
+image per input. The original pipeline then consumes those images, including
+its mandatory canvas adaptation, reference resizing, and encoder preprocessing.
+
+```text
+native-resolution image
+  -> input defaults + user overrides
+  -> one input transform
+  -> original generation pipeline and model-specific processing
+```
+
+These rules do not override internal VAE, CLIP/VLM, ControlNet, or pixel-patch preprocessing.
+`--ref-image-args` retains its existing meaning and runs after this input transform.
+
+## Inputs and defaults
+
+| `target` | Input | Default geometry | Indexed? |
+| --- | --- | --- | --- |
+| `init` | img2img image or video first frame | Center crop to the generation aspect ratio, then resize | No |
+| `end` | Video last frame | Center crop, then resize | No |
+| `mask` | Inpainting mask | Inherit init geometry; otherwise center crop, then resize | No |
+| `control` | Control image | Center crop, then resize | No |
+| `ref` | Reference images | Preserve source dimensions | Yes |
+| `ip-adapter` | IP-Adapter image | Preserve source dimensions | No |
+| `id` | PhotoMaker identity images | Preserve source dimensions | Yes |
+| `control-frame` | Control video frames | Center crop, then resize | Yes |
+
+Canvas defaults use the aligned generation dimensions. Reference, IP-Adapter,
+and identity inputs use their original dimensions unless overridden. Default
+resampling is nearest for images and nearest-exact for masks.
+
+These defaults are shared by CLI, server, and C API. Moving geometry out of
+the loaders replaces the previous CLI/server BOX/sRGB resizing, so default
+pixels are not guaranteed to match earlier builds.
+
+Reference video and audio preprocessing are outside these image rules.
+Preprocessing options apply to `img_gen` and `vid_gen`, not standalone upscale
+or ADetailer mode. ADetailer clears the user's rules for its internal crops.
+
+## Rules
+
+Rules are comma-separated `key=value` lists. Repeat the CLI option or separate
+rules with semicolons. Every rule requires a `target` and at least one option.
+Rule syntax and input compatibility are checked when image/video generation
+starts. Unknown keys, invalid values, duplicate keys in a rule, missing images,
+and out-of-range indices cause generation to fail with an error log.
+
+Omit `index` to configure every image of that type; otherwise use a zero-based
+index. CLI directory inputs follow filename order. Indexed rules override
+type-wide rules field by field, regardless of order. At equal specificity,
+the last value for a field wins. `auto` selects the input preset.
+
+| `mode` | Input transform |
+| --- | --- |
+| `auto` | Use the input's default geometry |
+| `none` | Keep source dimensions without resizing, cropping, or padding |
+| `stretch` | Resize to the target dimensions |
+| `crop` | Crop a target-sized rectangle without resizing; fail if the source is too small |
+| `crop-resize` | Crop to the target aspect ratio, then resize |
+| `fit-pad` | Fit the entire image inside the target dimensions, preserving aspect ratio, then pad |
+
+`width` and `height` must be specified together as positive integers. They
+override the input transform's dimensions, not the generation or encoder size.
+For a native-size preset, specifying dimensions without a mode selects stretch.
+`mode=none` with explicit dimensions different from the source is contradictory
+and is rejected.
+
+`anchor=center|top|bottom|left|right` selects crop/padding placement.
+`pad_color=#RRGGBB` or `#RRGGBBAA` selects padding, defaulting to opaque black.
+A grayscale mask uses the first color component.
+
+`filter=auto|nearest|nearest-exact|bilinear|bicubic|lanczos` selects resampling.
+`antialias=auto|true|false` enables antialiasing automatically for filtered
+downscaling; explicit true requires bilinear, bicubic, or Lanczos.
+Filtered RGBA resizing uses premultiplied alpha.
+
+`canny=true|false` enables edge detection for any supported image target,
+defaulting to `false`. It runs once after geometry, before the original
+generation pipeline, including with `mode=none`. Grayscale, grayscale-alpha,
+RGB, and RGBA inputs are supported; alpha is preserved.
+
+Each input has its own Canny setting. Indexed rules can enable or disable it
+for individual references, identity images, or video control frames.
+
+```sh
+--image-preprocess "target=init,mode=fit-pad,canny=true"
+--image-preprocess "target=ref,index=0,mode=none,canny=true"
+--image-preprocess "target=control-frame,index=2,canny=true"
+```
+
+Init and mask sources must have the same dimensions. The mask inherits the
+init crop, resize, and padding coordinates, while retaining its own filter,
+padding value, and Canny setting. Conflicting mask geometry is rejected. An
+omitted mask remains absent until the original pipeline creates its default mask.
+
+## Downstream behavior
+
+`mode=none` only skips the input geometry transform. For example:
+
+```sh
+--image-preprocess "target=init,mode=none" \
+--image-preprocess "target=ref,mode=none"
+```
+
+The init image is still adapted to the generation canvas by the original
+pipeline. Reference images still follow `--ref-image-args` and model-specific
+resizing. CLIP retains its fixed input dimensions and normalization. HiDream-O1
+retains its original pixel-reference and visual preprocessing.
+
+Existing sharing between consumers is preserved: for example, Wan img2video
+uses the same adapted first frame for VAE conditioning and CLIP. High-resolution
+passes reuse the prepared images and apply their original size adaptation;
+they do not apply the user's crop a second time.
+
+To disable reference resizing before VAE encoding, use
+`--ref-image-args "resize_before_vae=false"` or the server field
+`"ref_image_args": "resize_before_vae=false"`. This is separate from
+`target=ref,mode=none`, which only skips input geometry. Model constraints
+still apply.
+
+## Server requests
+
+Native image/video requests and SDAPI accept `image_preprocess` as a string or
+an array of rule strings:
+
+```json
+{
+  "image_preprocess": [
+    "target=init,mode=fit-pad,filter=bicubic",
+    "target=mask,filter=nearest-exact",
+    "target=ref,index=0,mode=none"
+  ]
+}
+```
+
+OpenAI-compatible requests accept it through
+`<sd_cpp_extra_args>{...}</sd_cpp_extra_args>` in the prompt.
+Request rules replace server-default rules. Generation metadata records the
+user rules; image encodings and channel conventions are unchanged.
+
+## C API
+
+Set `image_preprocess` on the existing image/video generation parameters.
+The `generate_image()` and `generate_video()` signatures are unchanged:
+
+```c
+sd_img_gen_params_t params;
+sd_img_gen_params_init(&params);
+/* Set prompt, original-resolution input images, and generation options. */
+params.image_preprocess.rules = "target=init,mode=crop-resize,filter=lanczos;"
+                                "target=mask,filter=nearest-exact";
+bool ok = generate_image(ctx, &params, &images, &count);
+```
+
+Both generation parameter initializers set `image_preprocess.rules` to `NULL`,
+selecting input presets. Rule strings are borrowed for the synchronous call.
+The library owns temporary transformed pixels; caller images and arrays are
+not modified. Add `canny=true` to the desired target's rule in
+`image_preprocess.rules` to enable Canny.
+
+The parameter structs have grown; applications and bindings must be rebuilt.
diff --git a/docs/int8_convrot.md b/docs/int8_convrot.md
index 3b65113d2d..04d9c4f73c 100644
--- a/docs/int8_convrot.md
+++ b/docs/int8_convrot.md
@@ -84,22 +84,23 @@ The floating-point output is reconstructed as
 Y[r, o] ~= A[r, o] * s_x[r] * s_w[o] + b[o]
 ```
 
-The packed runtime activation tensor contains the I8 activation rows and their floating-point row scales. Linear layers that share the same input and convrot group size reuse this packed tensor, avoiding repeated rotation and activation quantization within the graph.
+The packed runtime activation tensor contains the I8 activation rows and their floating-point row scales. Linear layers with an input scale of `1` that share the same input and convrot group size reuse this packed tensor, avoiding repeated rotation and activation quantization within the graph. For other input scales, activations are scaled before explicit convrot quantization, and the linear output is unscaled before adding bias.
 
 ## Backend support
 
 - CPU provides the portable regular Hadamard, activation quantization, INT8 matrix multiplication, and scale restoration implementations.
 - NVIDIA CUDA devices with compute capability 7.5 or newer use the native accelerated path. For H256, CUDA fuses the rotation, row-wise maximum reduction, and activation quantization. It uses cuBLAS for I8 x I8 to I32 GEMM and a CUDA kernel for scale restoration and bias addition.
-- Vulkan and other GPU backends do not currently have dedicated INT8 convrot kernels. They use the backend scheduler to fall back to CPU, which is expected to be substantially slower than the CUDA path.
+- AMD HIP devices in the CDNA, RDNA3 (including RDNA3.5), and RDNA4 families use the same INT8/H256 kernels with hipBLAS for I8 x I8 to I32 GEMM. Other AMD architectures fall back to CPU for INT8 matrix multiplication.
+- Vulkan provides native H256 activation quantization and INT8 matrix multiplication when the build and device support accelerated packed INT8 dot products. Unsupported configurations and GPU backends without these kernels use the backend scheduler to fall back to CPU.
 
 LoRA adapters are applied at runtime without modifying the INT8 weights. The INT8 convrot path computes the base linear output, while LoRA, LoHa, LoKr, and raw weight-difference adapters compute their output corrections from the original, unrotated activation and add them to the base output. `--lora-apply-mode auto` selects this path for models containing INT8 tensorwise weights. If `immediately` is requested, sd.cpp falls back to runtime application because merging an adapter would require dequantizing and rotating its weight update, then recalculating the per-row scales and requantizing the result.
 
-The dedicated CUDA convrot activation path currently requires a group size of `256`; other supported group sizes use CPU execution.
+The dedicated CUDA and HIP convrot activation paths currently require a group size of `256`; other supported group sizes use CPU execution.
 
 ## Example
 
 ComfyUI INT8 convrot safetensors can be passed to `--diffusion-model` without conversion:
 
 ```powershell
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\krea2_turbo_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" --steps 8 --cfg-scale 1 --diffusion-fa -v --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\krea2_turbo_int8_convrot.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" --steps 8 --cfg-scale 1 -v --offload-to-cpu
 ```
diff --git a/docs/ip_adapter.md b/docs/ip_adapter.md
index 76a651011b..27f9ae6896 100644
--- a/docs/ip_adapter.md
+++ b/docs/ip_adapter.md
@@ -48,7 +48,7 @@ sd-cli -m ..\models\sd_v1.5.safetensors --clip_vision ..\models\clip_vision_h.sa
 ## Example (SDXL)
 
 ```
-sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 6 --steps 25 --sampling-method dpm++2m --scheduler karras -W 1024 -H 1024 --diffusion-fa --vae-tiling
+sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\reference.png --ip-adapter-strength 0.8 -p "a woman, best quality" -n "lowres, bad anatomy" --cfg-scale 6 --steps 25 --sampling-method dpm++2m --scheduler karras -W 1024 -H 1024 --vae-tiling
 ```
 
 The SDXL VAE decode at 1024x1024 is memory heavy; add `--vae-tiling` (and
@@ -83,5 +83,5 @@ Add the usual Control Net options to keep the reference appearance while
 controlling the pose:
 
 ```
-sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\character.png --ip-adapter-strength 0.9 --control-net ..\models\OpenPoseXL2.safetensors --control-image ..\assets\pose.png --control-strength 0.8 -p "a character, side view" --cfg-scale 6 --steps 25 -W 1024 -H 1024 --diffusion-fa --vae-tiling
+sd-cli -m ..\models\sdxl.safetensors --clip_vision ..\models\clip_vision_h.safetensors --ip-adapter ..\models\ip-adapter_sdxl_vit-h.safetensors --ip-adapter-image ..\assets\character.png --ip-adapter-strength 0.9 --control-net ..\models\OpenPoseXL2.safetensors --control-image ..\assets\pose.png --control-strength 0.8 -p "a character, side view" --cfg-scale 6 --steps 25 -W 1024 -H 1024 --vae-tiling
 ```
diff --git a/docs/iris.md b/docs/iris.md
new file mode 100644
index 0000000000..322a3da1a4
--- /dev/null
+++ b/docs/iris.md
@@ -0,0 +1,24 @@
+# How to Use
+
+Iris-3B generates images directly in pixel space and uses Qwen3-VL-4B-Instruct as the text encoder. No VAE is required.
+
+## Download weights
+
+- Download Iris-3B
+    - safetensors: https://huggingface.co/speridlabs/iris-3b/tree/main (`model.safetensors` in the root directory)
+- Download Qwen3-VL-4B-Instruct
+    - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders
+    - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main
+
+## Examples
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\iris-3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf -p "a lovely cat" --cfg-scale 3 -H 1024 -W 1024
+```
+
+<img width="256" alt="iris-3b example" src="../assets/iris/example.png" />
+
+## Notes
+
+- Width and height must be multiples of 16. Do not pass `--vae`.
+- Captions are limited to 300 tokens including the assistant-turn suffix. Positive and negative prompts use the same template.
diff --git a/docs/krea2.md b/docs/krea2.md
index 10f8eab80e..67fad023ed 100644
--- a/docs/krea2.md
+++ b/docs/krea2.md
@@ -21,7 +21,7 @@ Krea2 uses a Krea2 diffusion transformer, the Wan2.1 VAE, and Qwen3-VL 4B as the
 ### Krea2
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Krea-2-Raw-Q8_0.gguf --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'"  --diffusion-fa -v --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Krea-2-Raw-Q8_0.gguf --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p "a lovely cat holding a sign says 'krea2.cpp'" -v --offload-to-cpu
 ```
 
 <img width="256" alt="Krea2 Raw example" src="../assets/krea2/example.png" />
diff --git a/docs/lens.md b/docs/lens.md
index 64dc594c47..09e9575ac9 100644
--- a/docs/lens.md
+++ b/docs/lens.md
@@ -22,7 +22,7 @@ Lens and Lens Turbo require an external GPT-OSS `tokenizer.json` matching the te
 ### Lens
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 5.0  -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" --diffusion-fa -v
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 5.0  -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" -v
 ```
 
 <img width="256" alt="Lens example" src="../assets/lens/example.png" />
@@ -30,7 +30,7 @@ Lens and Lens Turbo require an external GPT-OSS `tokenizer.json` matching the te
 ### Lens Turbo
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_turbo_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 1.0  -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" --diffusion-fa -v --steps 4
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\lens_turbo_bf16.safetensors --llm "..\models\text_encoders\gpt-oss-20b-UD-Q8_K_XL.gguf" --tokenizer ..\models\tokenizers\tokenizer_gpt_oss.json --vae ..\models\vae\flux2_ae.safetensors --cfg-scale 1.0  -p "A crystal dragon soaring through an aurora borealis sky, its entire body made of transparent faceted crystal refracting the green and purple aurora light into rainbow spectra, ice particles trailing from its wings, high fantasy digital art" -v --steps 4
 ```
 
 <img width="256" alt="Lens Turbo example" src="../assets/lens/turbo_example.png" />
diff --git a/docs/lingbot_video.md b/docs/lingbot_video.md
index d1248bd5bc..b92165ab5f 100644
--- a/docs/lingbot_video.md
+++ b/docs/lingbot_video.md
@@ -28,5 +28,5 @@ Lingbot Video uses a Lingbot diffusion transformer, the Wan2.1 VAE, and Qwen3-VL
 ### LingBot-Video T2V
 
 ```sh
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\lingbot-video-dens-1.3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p '{"caption":{"comprehensive_description":"A lovely cat sits comfortably on a soft cushion near a sunlit window, looking calm, gentle, and adorable. The cat has soft fluffy fur, bright expressive eyes, small rounded ears, delicate whiskers, and a relaxed posture. Warm daylight falls across the cat from one side, creating soft highlights on the fur and a cozy glow around the scene. The background is softly blurred, showing hints of a peaceful indoor home environment with warm tones and gentle natural light. The overall atmosphere is cute, tender, serene, and photorealistic, emphasizing the cat''s charming appearance, soft texture, and affectionate presence.","camera_info":{"color":"Warm","frame_size":"Close Up","shot_type_angle":"Eye level","lens_size":"Medium Lens","composition":"Centered balanced","lighting":"Soft light","lighting_type":"Daylight"},"world_knowledge":[],"prominent_elements":[{"name":"lovely cat","description":"A cute and gentle domestic cat with soft fluffy fur, expressive eyes, and a calm relaxed presence.","location":"center of the frame","relative_size":"large","shape_and_color":"Small animal body with rounded face, triangular ears, bright eyes, and soft fur in warm natural tones","texture":"soft, fluffy, silky","appearance_details":"The cat has clean well-groomed fur, delicate whiskers, small ears, a cute nose, and bright attentive eyes. Its expression appears calm, affectionate, and slightly curious.","relationship":"Acts as the main subject and emotional focal point of the scene.","orientation":"facing the camera","pose":"sitting comfortably with a relaxed posture","expression":"gentle, adorable, calm, slightly curious","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"cat eyes","description":"Bright expressive eyes that give the cat a sweet and affectionate appearance.","location":"upper center of the cat face","relative_size":"small","shape_and_color":"Round almond-like eyes with glossy reflections","texture":"clear, glossy, reflective","appearance_details":"The eyes catch the soft daylight, creating small natural highlights that make the cat look vivid and alive.","relationship":"Enhance the emotional charm and cuteness of the cat.","orientation":"looking toward the camera","pose":"","expression":"soft and attentive","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"soft cushion","description":"A comfortable cushion or blanket where the cat is resting.","location":"bottom portion of the frame","relative_size":"medium","shape_and_color":"Soft rounded fabric surface in light warm neutral tones","texture":"plush, fabric, cozy","appearance_details":"The cushion gently supports the cat and adds a comfortable home-like feeling to the scene.","relationship":"Provides a cozy resting place for the cat.","orientation":"horizontal","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"sunlit indoor background","description":"A softly blurred indoor background with warm daylight and peaceful home atmosphere.","location":"behind the cat, filling the upper and side areas of the frame","relative_size":"large","shape_and_color":"Soft abstract shapes in warm beige, cream, and pale golden tones","texture":"soft, blurry, bokeh-like","appearance_details":"The background is intentionally out of focus, keeping attention on the cat while creating a cozy and serene mood.","relationship":"Provides a warm and gentle environment that supports the cute domestic scene.","orientation":"upright","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":"","is_cluster":true,"number_of_objects":"numerous"}]}}' -n '{"universal_negative":{"visual_quality":["low quality","worst quality","blurry","pixelated","jpeg artifacts","low resolution","unstable color","color flicker","underexposed","overexposed","invisible subject","subject hidden in darkness"],"artistic_style":["painting","illustration","drawing","cartoon","3d render","cgi","sketch","digital art"],"composition_and_content":["text","watermark","signature","logo","subtitles","pillarboxed","side bars","portrait image in landscape frame"],"temporal_and_motion_stability":["flickering","jittery","motion blur","temporal inconsistency","warping","morphing","incoherent motion","unnatural movement","static object with sudden jump","frame-to-frame inconsistency"],"material_and_structure":["plastic-like glass","unrealistic texture","deformed bottle","liquid freezing improperly","distorted reflections"]}}' --diffusion-fa --offload-to-cpu --cfg-scale 3 --video-frames 33 -v
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\lingbot-video-dens-1.3b.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\wan_2.1_vae.safetensors -p '{"caption":{"comprehensive_description":"A lovely cat sits comfortably on a soft cushion near a sunlit window, looking calm, gentle, and adorable. The cat has soft fluffy fur, bright expressive eyes, small rounded ears, delicate whiskers, and a relaxed posture. Warm daylight falls across the cat from one side, creating soft highlights on the fur and a cozy glow around the scene. The background is softly blurred, showing hints of a peaceful indoor home environment with warm tones and gentle natural light. The overall atmosphere is cute, tender, serene, and photorealistic, emphasizing the cat''s charming appearance, soft texture, and affectionate presence.","camera_info":{"color":"Warm","frame_size":"Close Up","shot_type_angle":"Eye level","lens_size":"Medium Lens","composition":"Centered balanced","lighting":"Soft light","lighting_type":"Daylight"},"world_knowledge":[],"prominent_elements":[{"name":"lovely cat","description":"A cute and gentle domestic cat with soft fluffy fur, expressive eyes, and a calm relaxed presence.","location":"center of the frame","relative_size":"large","shape_and_color":"Small animal body with rounded face, triangular ears, bright eyes, and soft fur in warm natural tones","texture":"soft, fluffy, silky","appearance_details":"The cat has clean well-groomed fur, delicate whiskers, small ears, a cute nose, and bright attentive eyes. Its expression appears calm, affectionate, and slightly curious.","relationship":"Acts as the main subject and emotional focal point of the scene.","orientation":"facing the camera","pose":"sitting comfortably with a relaxed posture","expression":"gentle, adorable, calm, slightly curious","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"cat eyes","description":"Bright expressive eyes that give the cat a sweet and affectionate appearance.","location":"upper center of the cat face","relative_size":"small","shape_and_color":"Round almond-like eyes with glossy reflections","texture":"clear, glossy, reflective","appearance_details":"The eyes catch the soft daylight, creating small natural highlights that make the cat look vivid and alive.","relationship":"Enhance the emotional charm and cuteness of the cat.","orientation":"looking toward the camera","pose":"","expression":"soft and attentive","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"soft cushion","description":"A comfortable cushion or blanket where the cat is resting.","location":"bottom portion of the frame","relative_size":"medium","shape_and_color":"Soft rounded fabric surface in light warm neutral tones","texture":"plush, fabric, cozy","appearance_details":"The cushion gently supports the cat and adds a comfortable home-like feeling to the scene.","relationship":"Provides a cozy resting place for the cat.","orientation":"horizontal","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":""},{"name":"sunlit indoor background","description":"A softly blurred indoor background with warm daylight and peaceful home atmosphere.","location":"behind the cat, filling the upper and side areas of the frame","relative_size":"large","shape_and_color":"Soft abstract shapes in warm beige, cream, and pale golden tones","texture":"soft, blurry, bokeh-like","appearance_details":"The background is intentionally out of focus, keeping attention on the cat while creating a cozy and serene mood.","relationship":"Provides a warm and gentle environment that supports the cute domestic scene.","orientation":"upright","pose":"","expression":"","clothing":"","gender":"","skin_tone_and_texture":"","is_cluster":true,"number_of_objects":"numerous"}]}}' -n '{"universal_negative":{"visual_quality":["low quality","worst quality","blurry","pixelated","jpeg artifacts","low resolution","unstable color","color flicker","underexposed","overexposed","invisible subject","subject hidden in darkness"],"artistic_style":["painting","illustration","drawing","cartoon","3d render","cgi","sketch","digital art"],"composition_and_content":["text","watermark","signature","logo","subtitles","pillarboxed","side bars","portrait image in landscape frame"],"temporal_and_motion_stability":["flickering","jittery","motion blur","temporal inconsistency","warping","morphing","incoherent motion","unnatural movement","static object with sudden jump","frame-to-frame inconsistency"],"material_and_structure":["plastic-like glass","unrealistic texture","deformed bottle","liquid freezing improperly","distorted reflections"]}}' --offload-to-cpu --cfg-scale 3 --video-frames 33 -v
 ```
diff --git a/docs/llada_image.md b/docs/llada_image.md
index 9b24158be9..d939e78cea 100644
--- a/docs/llada_image.md
+++ b/docs/llada_image.md
@@ -122,7 +122,6 @@ save_file(merged, "llada_connectors.safetensors")
   --height 1024 \
   --steps 4 \
   --cfg-scale 1.0 \
-  --diffusion-fa \
   --output output.png
 ```
 
@@ -145,7 +144,7 @@ only in the positive branch. Editing requires connectors that include the SigVQ
   unchanged instead of applying the instruction; LLaDA-Image-Turbo edits correctly at both.
 - Editing runs the reference and the target in one sequence, so it needs roughly twice the
   tokens of text to image at the same size. On 12 GB, editing at 1024x1024 needs
-  `--diffusion-fa`; without it the diffusion graph does not fit.
+  flash attention; without it the diffusion graph does not fit.
 - The weights total about 16 GB, but segmented execution streams them, so a much smaller
   budget works. At 512x512, `--max-vram 6` costs almost nothing over unconstrained execution,
   and `--max-vram 3` still produces byte-identical output at roughly 2.5x the time.
diff --git a/docs/longcat_image.md b/docs/longcat_image.md
index c628492277..ae730b4727 100644
--- a/docs/longcat_image.md
+++ b/docs/longcat_image.md
@@ -24,7 +24,7 @@ LongCat uses quoted text for character-level text rendering. Put target text ins
 ### LongCat Image
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\LongCat-Image-Q4_K_M.gguf --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf -p "a lovely cat holding a sign says 'longcat.cpp'" --cfg-scale 5.0 --sampling-method euler --flow-shift 3 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\LongCat-Image-Q4_K_M.gguf --vae ..\models\vae\ae.sft --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf -p "a lovely cat holding a sign says 'longcat.cpp'" --cfg-scale 5.0 --sampling-method euler --flow-shift 3 -v --offload-to-cpu
 ```
 
 <img alt="longcat example" src="../assets/longcat/example.png" />
diff --git a/docs/ltx2.md b/docs/ltx2.md
index 9caa974113..e42698ffb3 100644
--- a/docs/ltx2.md
+++ b/docs/ltx2.md
@@ -55,7 +55,7 @@ To run the text encoder quantized, convert it once with sd-cli:
 ### LTX-2.3 dev T2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 33 --fps 24 -o t2v.webm
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --offload-to-cpu --video-frames 33 --fps 24 -o t2v.webm
 ```
 
 <video
@@ -67,7 +67,7 @@ To run the text encoder quantized, convert it once with sd-cli:
 ### LTX-2.3 dev I2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v  -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 33 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v  -W 1280 -H 720 --offload-to-cpu --video-frames 33 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm
 ```
 
 <video
@@ -79,7 +79,7 @@ To run the text encoder quantized, convert it once with sd-cli:
 ### LTX-2.3 dev FLF2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v  -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 33 --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png -o flf2v.webm
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors  -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v  -W 1280 -H 720 --offload-to-cpu --video-frames 33 --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png -o flf2v.webm
 ```
 
 <video
@@ -95,7 +95,7 @@ LTX spatial latent upscale runs a model-backed x2 latent upsampler between the l
 Put `ltx-2.3-spatial-upscaler-x2-1.1.safetensors` under the directory passed to `--hires-upscalers-dir`, then use the model name without path or extension in `--hires-upscaler`.
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.3-spatial-upscaler-x2-1.1 --hires --hires-steps 4 -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v  -W 640 -H 360 --diffusion-fa --offload-to-cpu --video-frames 33 -i ..\assets\ernie_image\turbo_example.png -o hires_i2v.webm
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\ltx-2.3-22b-dev-UD-Q4_K_M.gguf --vae ..\models\vae\ltx-2.3-22b-dev_video_vae.safetensors --audio-vae ..\models\vae\ltx-2.3-22b-dev_audio_vae.safetensors --llm ..\models\text_encoders\gemma-3-12b-it-qat-UD-Q4_K_XL.gguf --embeddings-connectors ..\models\text_encoders\ltx-2.3-22b-dev_embeddings_connectors.safetensors --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.3-spatial-upscaler-x2-1.1 --hires --hires-steps 4 -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v  -W 640 -H 360 --offload-to-cpu --video-frames 33 -i ..\assets\ernie_image\turbo_example.png -o hires_i2v.webm
 ```
 
 By default, the hires refine pass uses the main sampler and scheduler, then trims the second-pass sigma schedule by `--hires-denoising-strength` (`0.7` by default). To reproduce a ComfyUI-style explicit refine schedule, pass custom hires sigmas:
@@ -113,13 +113,13 @@ By default, the hires refine pass uses the main sampler and scheduler, then trim
 ### LTX-2.5 dev T2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below" --cfg-scale 3.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 121 --fps 24 -o t2v.webm
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "A wide aerial shot of a red vintage convertible driving along a coastal cliff road at sunset, waves crashing below" --cfg-scale 3.0 --sampling-method euler -v -n "worst quality, low quality, blurry, distorted, artifacts" -W 1280 -H 720 --offload-to-cpu --video-frames 121 --fps 24 -o t2v.webm
 ```
 
 ### LTX-2.5 dev I2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "a lovely cat blinking slowly, gentle camera push in" --cfg-scale 3.0 --sampling-method euler -v -W 1280 -H 720 --diffusion-fa --offload-to-cpu --video-frames 121 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf -p "a lovely cat blinking slowly, gentle camera push in" --cfg-scale 3.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --video-frames 121 -i ..\assets\ernie_image\turbo_example.png -o i2v.webm
 ```
 
 ### LTX-2.5 spatial latent upscale
@@ -129,7 +129,7 @@ Works exactly like the LTX-2.3 upscaler described below; put
 pass its name without path or extension to `--hires-upscaler`.
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0 --hires --hires-steps 6 -p "a lovely cat" --cfg-scale 3.0 --sampling-method euler -v -W 640 -H 360 --diffusion-fa --offload-to-cpu --video-frames 121 -o hires_t2v.webm
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\ltx-2.5-22b-dev-transformer-Q8_0.gguf --vae ..\models\vae\ltx-2.5-video-vae-conv-bf16.safetensors --audio-vae ..\models\vae\ltx-2.5-audio-vae-bf16.safetensors --llm ..\models\text_encoders\gemma4-12b-with-proj-ltx-2.5-Q8_0.gguf --hires-upscalers-dir ..\models\latent_upscale_models --hires-upscaler ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0 --hires --hires-steps 6 -p "a lovely cat" --cfg-scale 3.0 --sampling-method euler -v -W 640 -H 360 --offload-to-cpu --video-frames 121 -o hires_t2v.webm
 ```
 
 ## Not implemented
diff --git a/docs/mage_flow.md b/docs/mage_flow.md
index 1d8c9a4cd7..0adef4afb6 100644
--- a/docs/mage_flow.md
+++ b/docs/mage_flow.md
@@ -4,20 +4,8 @@
 
 ## Download weights
 
-- Download Mage-Flow
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow/tree/main/transformer
-- Download Mage-Flow-Base
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Base/tree/main/transformer
-- Download Mage-Flow-Turbo
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Turbo/tree/main/transformer
-- Download Mage-Flow-Edit
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Edit/tree/main/transformer
-- Download Mage-Flow-Edit-Turbo
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Edit-Turbo/tree/main/transformer
-- Download Mage-Flow-Edit-Base
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow-Edit-Base/tree/main/transformer
-- Download Mage-Flow vae
-    - safetensors: https://huggingface.co/microsoft/Mage-Flow/tree/main/vae
+- Download Mage-Flow diffusion from https://huggingface.co/Comfy-Org/Mage-Flow/tree/main/diffusion_models
+- Download Mage-Flow vae from https://huggingface.co/Comfy-Org/Mage-Flow/tree/main/vae
 - Download Qwen3-VL 4B
     - safetensors: https://huggingface.co/Comfy-Org/Krea-2/tree/main/text_encoders
     - gguf: https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct-GGUF/tree/main
@@ -27,7 +15,7 @@
 Use 30 steps for Base models and 4 steps with `--cfg-scale 1` for Turbo models. Image dimensions must be multiples of 16; the official checkpoints are trained for native resolutions from 512 to 2048 pixels.
 
 ```bash
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Mage-Flow-Turbo.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\mage_vae.safetensors -p "a lovely cat holding a sign says 'mage.cpp'" --cfg-scale 1.0 --steps 4 --diffusion-fa -v --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Mage-Flow-Turbo.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --vae ..\models\vae\mage_vae.safetensors -p "a lovely cat holding a sign says 'mage.cpp'" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu
 ```
 
 <img width="256" alt="Mage-Flow example" src="../assets/mage_flow/example.png" />
@@ -39,7 +27,7 @@ Mage-Flow-Edit accepts one or more reference images. The default `mage_flow` ref
 For the Turbo edit checkpoint, use 4 steps and `--cfg-scale 1`.
 
 ```bash
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Mage-Flow-Edit.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf --vae ..\models\vae\mage_vae.safetensors -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'mage.cpp'" --cfg-scale 4.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\Mage-Flow-Edit.safetensors --llm ..\models\text_encoders\Qwen3-VL-4B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\Qwen3-VL-4B-Instruct-mmproj-BF16.gguf --vae ..\models\vae\mage_vae.safetensors -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'mage.cpp'" --cfg-scale 4.0 --sampling-method euler -v --offload-to-cpu
 ```
 
 <img width="256" alt="Mage-Flow-Edit example" src="../assets/mage_flow/edit_example.png" />
diff --git a/docs/ming_image.md b/docs/ming_image.md
new file mode 100644
index 0000000000..c242559212
--- /dev/null
+++ b/docs/ming_image.md
@@ -0,0 +1,24 @@
+# Ming-Image
+
+[Ming-Image](https://github.com/inclusionAI/Ming-Image) 0.1 Design uses a 6B diffusion transformer (DiT), Ling-mini-2.0 for text conditioning, and the Ming-Image VAE. Text-to-image generation with RGBA output is supported.
+
+## Download weights
+
+- Download Ming-Image 0.1 Design DiT
+    - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/diffusion_models
+- Download Ling-mini-2.0 BF16
+    - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/text_encoders
+- Download Ming-Image VAE
+    - safetensors: https://huggingface.co/Comfy-Org/Ming-Image/tree/main/vae
+- Download Ling tokenizer
+    - tokenizer.json: https://huggingface.co/inclusionAI/Ming-Image-0.1-Design/blob/main/mllm/tokenizer.json
+
+The example below uses `ming_image_0.1_design_bf16.safetensors` for the DiT. You can also use `ming_image_0.1_design_int8_convrot.safetensors` with [INT8 convrot support](int8_convrot.md). Use the BF16 text encoder.
+
+## Text-to-image
+
+Pass the Ling `tokenizer.json` with `--tokenizer` and use the matching Ming-Image VAE. Image dimensions must be multiples of 16. Save the output as PNG to preserve the alpha channel.
+
+```bash
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\ming_image_0.1_design_bf16.safetensors --llm ..\models\text_encoders\ming_image_0.1_ling_mini_2.0_bf16.safetensors --vae ..\models\vae\ming_image_vae_bf16.safetensors --tokenizer ..\models\text_encoders\tokenizer.json -p "A cheerful orange cat sticker, transparent background" --width 1024 --height 1024 --steps 12 --cfg-scale 1 --sampling-method euler -v --offload-to-cpu -o ming_image.png
+```
diff --git a/docs/minimax_h3.md b/docs/minimax_h3.md
index 82d0e2ffb4..391850c2c2 100644
--- a/docs/minimax_h3.md
+++ b/docs/minimax_h3.md
@@ -38,7 +38,7 @@ are detected from their weights.
 ## Text-to-audio-video
 
 ```sh
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten’s energetic surfing." --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --fps 24 --video-frames 56
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "A cute American Shorthair silver tabby kitten surfs on a tropical ocean wave, riding a white surfboard with the clear text 'sd.cpp' on it. Cinematic tracking shot, realistic water, bright sunlight, smooth motion, and consistent character appearance. Add upbeat tropical surf-rock background music with cheerful drums and guitar, synchronized with the kitten’s energetic surfing." --cfg-scale 1.0 -v -W 864 -H 480 --offload-to-cpu --rng cpu --fps 24 --video-frames 56
 ```
 
 <video src=../assets/minimax-h3/t2av.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -46,12 +46,95 @@ are detected from their weights.
 Omitting `--audio-vae` still runs the joint diffusion model but produces video without a
 decoded audio track.
 
+On CUDA and ROCm, `--diffusion-fa` also enables flash attention in the video VAE decoder (a ViT),
+which removes most of its attention cost without touching the text encoder. Set
+`SD_H3_VAE_FLASH_ATTN=0` to decode with the previous mul_mat + softmax attention. Other backends
+keep that attention by default (on Vulkan the flash attention decode was slower);
+`SD_H3_VAE_FLASH_ATTN=1` turns it on there.
+
+On NVIDIA GPUs from Ampere on, the unmasked DiT and VAE attention runs a pipelined
+long-sequence variant of the ggml-cuda flash attention kernel (up to 4x faster at H3
+shapes; the arithmetic per output element is unchanged). `GGML_CUDA_FA_LONGSEQ=0` restores the
+stock kernel. `GGML_CUDA_FA_LONGSEQ_NCOLS=128` opts into a wider tile that is faster on A100 and L4
+but not bit-identical.
+
+Builds configured with `-DGGML_CUDA_CUDNN=ON` (the Linux CUDA prebuilt is) can run that unmasked
+attention, and the `--sage-attn` attention, through cuDNN's fused attention. By default it replaces
+the flash attention kernel on Ada (sm89) and B200-class (sm100) GPUs and the sage kernel on sm100 only,
+where it measured faster; on an RTX 3090 (sm86) both existing kernels were faster, on an RTX 6000 Ada
+the sage kernel was, and other architectures were not measured.
+Only `cudnn.h` is needed at build time (`GGML_CUDA_CUDNN_INCLUDE_DIR`; the header-only
+cudnn-frontend v1.26.0 is downloaded by CMake unless `GGML_CUDA_CUDNN_FRONTEND_DIR` points at a
+checkout). `libcudnn.so.9` is opened when the first attention op runs and is not shipped: put a cuDNN 9
+built for the same CUDA major version as the binary (cuDNN 9 for CUDA 12 for the prebuilt) on the
+library path or beside the binary, or point `GGML_CUDA_CUDNN_LIB` at it (cuDNN 9.27 needs nothing
+else; older 9.x releases build these kernels with NVRTC and also need that CUDA major's `libnvrtc`).
+Without a loadable cuDNN, on Turing and older, for attention calls under about a million scores
+(where the extra conversions cost more than cuDNN saves) and for any shape cuDNN declines, the
+existing kernels run. The first call of each attention shape builds a cuDNN plan (about 0.4 to 1.3 s
+on a B200, logged once). cuDNN runs F16 with F32 accumulation: for the H3 DiT at 960x544x124 it
+takes about 8 ms per attention call on a B200 against 28 ms for the sage kernel and 43 ms for the
+ggml flash attention kernel, and it is closer to an exact (fp64) reference than either, so frames
+and audio differ slightly from the previous kernels. `GGML_CUDA_CUDNN_ATTN=0` restores them and
+`GGML_CUDA_CUDNN_ATTN=1` uses cuDNN for the flash attention op on any Ampere or newer GPU;
+`GGML_CUDA_CUDNN_SAGE=0` / `=1` does the same for the sage op only, and
+`GGML_CUDA_CUDNN_ATTN_BF16=1` runs cuDNN in BF16 (less accurate, same speed).
+
+The video VAE decodes one 16x16 latent tile (256x256 pixels) per decoder graph by default.
+`SD_H3_VAE_TILE=N` uses N x N latent tiles instead, overriding `--vae-tile-size` (20 decodes
+about 0.7 s faster at 960x544x124 on B200); the tile seams move, so the frames differ from the
+default (36 dB PSNR at 20) and it is opt-in.
+`SD_H3_VAE_TILE_BATCH=auto` puts several tiles into one graph, sized from free device memory (at
+most 4 unless `SD_H3_VAE_TILE_BATCH_MAX` raises it), and `SD_H3_VAE_TILE_BATCH=N` forces N; the
+batched projections can round differently from the per-tile decode on some GPUs, so it is
+opt-in. The decoder weights stay on the device across temporal chunks
+(`SD_H3_VAE_KEEP_RESIDENT=0` releases them after every chunk), and the decoder blocks use a
+table-based rotary embedding and a fused SwiGLU (`SD_H3_VAE_GRAPH_OPT=0` restores the previous
+graph). Each batched tile still goes through its own attention call. With one tile per graph and
+flash attention, each decoder attention normalises q/k in place on the projection and one fused
+RoPE op per tensor writes the head-major Q (F32) and K/V (F16) that the attention kernel reads,
+replacing the table RoPE, chunk copies, permutes and casts; the frames are bit-identical
+(`SD_H3_VAE_FUSED_QKV=0` restores the unfused graph). On CUDA the q/k RMS norm runs inside that
+RoPE op with the same reduction as the standalone norm (`SD_H3_VAE_FUSED_QK_NORM=0` keeps it
+separate).
+
+The host side of the decode overlaps the device: each tile's blend into the frame runs on a
+worker thread while the next tile computes (`SD_TILE_ASYNC_MERGE=0` blends inline; this applies
+to every tiled VAE decode), each temporal chunk is trimmed, cross-faded and copied into the final
+frames on a worker thread while the next chunk decodes (`SD_H3_VAE_ASYNC_ASSEMBLY=0` restores
+the concatenate-at-the-end path), the rotary tables are built once per tile shape, and a tile
+that allocates nothing new reuses the device free-memory reading taken earlier in the decode
+instead of querying the device for every capacity check (`SD_H3_VAE_REUSE_MEMQUERY=0`). The
+decoded frames are bit-identical either way.
+
+The DiT blocks use fused ggml ops (CPU and CUDA) for the work around the matmuls and attention:
+partial RoPE with the attention relayout and the K/V scale and F16 cast, per-segment adaLN
+modulation and gated residuals written in place, and the MLP Linear scales folded into those ops
+and the swiglu. The result is bit-identical to the unfused graph. `SD_H3_GRAPH_FAST=0` restores
+the unfused graph; `SD_H3_FAST_QKV=0`, `SD_H3_FAST_MLP=0`, `SD_H3_FAST_SEGMENTS=0` and
+`SD_H3_FAST_VIEWS=0` turn off one part each. With `--sage-attn` the same fused RoPE op writes the
+F32 Q/K and F16 V layout the sage kernel reads (K and V carry the kv scale), so the sage path no
+longer pays for the chunk / slice / rope / concat / scale / cast chain; the output is bit-identical
+to the unfused sage graph. `SD_H3_FAST_SAGE_QKV=0` restores that chain.
+
+The audio VAE's anti-aliased activations run their up/down-sampling filters as direct F32
+depthwise convolutions (`CONV_2D_DW`) instead of an F16 im2col plus a matrix-vector product,
+which makes the audio decode several times faster. The input is no longer rounded to F16, so
+the waveform differs slightly from the previous graph (about 41 dB SNR on a 5 s clip).
+`SD_H3_AUDIO_DIRECT_DW=0` restores the previous graph; backends without `CONV_2D_DW` keep it.
+
+The long-sequence flash attention kernel, the fused cuBLAS epilogues and the fused DiT ops come from
+the ggml patches in `scripts/unsloth/ggml-patches`, which the Unsloth prebuilt binaries carry. A
+source build gets them by applying the patches to the `ggml` submodule before configuring
+(`for p in scripts/unsloth/ggml-patches/*.patch; do git -C ggml apply "../$p"; done`); without
+them the build uses the stock ggml kernels and the unfused DiT graph.
+
 ## First/last-frame conditioning
 
 Add `--init-img` for I2VA, or both `--init-img` and `--end-img` for FL2VA:
 
 ```sh
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "a lovely cat" -i ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --fps 24 --video-frames 56
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_fl2va-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "a lovely cat" -i ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 -v -W 864 -H 480 --offload-to-cpu --rng cpu --fps 24 --video-frames 56
 ```
 
 <video src=../assets/minimax-h3/i2av.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -62,7 +145,7 @@ Ref2VA accepts any combination of reference images, reference videos, paired
 video soundtracks, and standalone audio references:
 
 ```sh
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_ref2va_pruned-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "Use the cat from <Picture 1> as the main character. Keep the cat’s appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat’s face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat’s head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat’s fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat’s identity unchanged." -r ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 -v -W 864 -H 480 --diffusion-fa --offload-to-cpu --rng cpu --fps 24 --video-frames 56
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\minimax_h3_ref2va_pruned-Q4_K_M.gguf --vae ..\models\vae\minimax_h3_video_vae_fp16.safetensors --audio-vae ..\models\vae\minimax_h3_audio_vae_fp32.safetensors --llm ..\models\text_encoders\qwen3vl_32b_minimax_h3-Q4_K_M.gguf -p "Use the cat from <Picture 1> as the main character. Keep the cat’s appearance, fur color, facial features, and identity consistent with the reference image. Create a 2-second cinematic video: start with an extreme close-up shot of the cat’s face, focusing on its cute expression and detailed fur texture. The camera slowly rotates around the cat’s head, creating a dynamic reveal. Then smoothly pull back and zoom out to reveal the full scene: the cat is standing confidently on a surfboard, riding ocean waves. Water splashes around the board, sea breeze gently moves the cat’s fur, and the cat maintains a cute and fearless expression while surfing. Smooth camera movement, cinematic orbit shot, seamless zoom-out transition, low-angle wide shot, realistic ocean environment, golden sunlight, dynamic waves, high-quality realistic style, natural motion, no distortion, keep the cat’s identity unchanged." -r ..\assets\ernie_image\turbo_example.png --cfg-scale 1.0 -v -W 864 -H 480 --offload-to-cpu --rng cpu --fps 24 --video-frames 56
 ```
 
 <video src=../assets/minimax-h3/r2av.mp4 controls="controls" muted="muted" type="video/mp4"></video>
diff --git a/docs/minit2i.md b/docs/minit2i.md
index 7e120b38c0..9f813a962f 100644
--- a/docs/minit2i.md
+++ b/docs/minit2i.md
@@ -43,6 +43,5 @@ MiniT2I uses a MiniT2I diffusion transformer and `google/flan-t5-large` as the t
   --height 512 \
   --seed 42 \
   --sampling-method euler \
-  --diffusion-fa \
   --output minit2i_cuda.png
 ```
diff --git a/docs/moss_image.md b/docs/moss_image.md
new file mode 100644
index 0000000000..27ab26f804
--- /dev/null
+++ b/docs/moss_image.md
@@ -0,0 +1,37 @@
+# How to Use
+
+MOSS-Image uses a 5B diffusion transformer, the FLUX.2 VAE, and Qwen3.5 9B as the LLM text encoder. Both Base and SFT are supported. Prompt enhancement is not included.
+
+## Download weights
+
+- Download MOSS-Image Base or SFT
+    - safetensors: https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/diffusion_models
+    - gguf:
+        - Base: https://huggingface.co/leejet/MOSS-Image-Base-GGUF/tree/main
+        - SFT: https://huggingface.co/leejet/MOSS-Image-SFT-GGUF/tree/main
+- Download vae
+    - safetensors: https://huggingface.co/Comfy-Org/MOSS-Image/tree/main/vae
+- Download Qwen3.5 9B
+    - gguf: https://huggingface.co/unsloth/Qwen3.5-9B-GGUF/tree/main
+- Download `tokenizer.json`
+    - https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base/tree/main/tokenizer
+
+Pass the external `tokenizer.json` with `--tokenizer`; Qwen3.5 uses a different vocabulary from the embedded Qwen tokenizer.
+
+## Examples
+
+### MOSS-Image Base
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\moss_image_base_int8_convrot_attn.safetensors --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json --cfg-scale 3.0 -p "a lovely cat holding a sign says 'moss-image.cpp'" -v --offload-to-cpu -H 1024 -W 1024
+```
+
+### MOSS-Image SFT
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\moss_image_sft_int8_convrot_attn.safetensors --llm ..\models\text_encoders\Qwen3.5-9B-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors --tokenizer ..\models\tokenizers\tokenizer_qwen3.5.json --cfg-scale 3.0 -p "a lovely cat holding a sign says 'moss-image.cpp'" -v --offload-to-cpu -H 1024 -W 1024
+```
+
+<img src="../assets/moss/sft.png" />
+
+Use dimensions divisible by 16. The default discrete scheduler uses the upstream time shift of 5. Use `--flow-shift` to override it. The negative CFG branch skips DiT layer 22 (zero-based), matching the upstream pipeline.
diff --git a/docs/ovis_image.md b/docs/ovis_image.md
index 235e19b285..c561ff54b0 100644
--- a/docs/ovis_image.md
+++ b/docs/ovis_image.md
@@ -13,7 +13,7 @@
 ## Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ovis_image-Q4_0.gguf --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\ovis_2.5.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ovis_image-Q4_0.gguf --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\ovis_2.5.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu
 ```
 
 <img alt="ovis image example" src="../assets/ovis_image/example.png" />
\ No newline at end of file
diff --git a/docs/performance.md b/docs/performance.md
index 2166720200..18491039eb 100644
--- a/docs/performance.md
+++ b/docs/performance.md
@@ -5,10 +5,15 @@ eg.:
  - flux 768x768 ~600mb
  - SD2 768x768 ~1400mb
 
-For most backends, it slows things down, but for cuda it generally speeds it up too.
-At the moment, it is only supported for some models and some backends (like cpu, cuda/rocm, metal).
-
-Run by adding `--diffusion-fa` to the arguments and watch for:
+`--flash-attn on|off|auto` controls flash attention. Its effect on speed depends
+on GPU architecture, backend, driver, model, and tensor shapes. Automatic
+selection uses backend rules and, for some paths, small probes; it does not
+guarantee the fastest end-to-end generation. Compare `on` and `off` with the
+same model and generation settings over repeated runs, separating startup/probe
+time from steady-state generation time. See [backend selection](backend.md)
+for automatic selection rules and per-module overrides.
+
+Watch for:
 ```
 [INFO ] stable-diffusion.cpp:312  - Using flash attention in the diffusion model
 ```
@@ -21,6 +26,87 @@ CPU fallback. It excludes weights and cache buffers. Within a runner lifecycle,
 the summary is printed only on the first graph or when backend capacities or the
 segment count change.
 
+## Tune direct convolution.
+
+`--conv-direct on|off|auto` selects direct convolution or the im2col-based
+path. Performance depends on GPU architecture, backend, driver, model, and
+tensor shapes, so enabling direct convolution can either improve or reduce speed.
+
+Automatic selection uses small benchmarks on Vulkan and implicit GEMM checks
+on other backends. It does not guarantee the fastest end-to-end generation.
+Compare `on` and `off` with the same model and generation settings over
+repeated runs, keeping flash attention unchanged and separating startup/probe
+time from steady-state generation time. See [backend selection](backend.md)
+for automatic selection rules and per-module overrides.
+
+## Run conditional and unconditional CFG in one batched UNet forward.
+
+For UNet models, the conditional and unconditional guidance branches are
+concatenated into a single batch of two and run through one UNet forward per
+step instead of two separate forwards. This is enabled by default whenever the
+run qualifies for it.
+
+Use `--batched-cfg off` to force separate conditional and unconditional
+forwards.
+
+## Use VAE tiling to reduce encode and decode memory usage.
+
+`--vae-tiling` enables spatial tiling for both VAE encoding and decoding. The
+default tile size is 256x256 **image pixels**, independent of the VAE scale factor:
+
+```shell
+--vae-tiling --vae-tile-size 256x256 --vae-tile-overlap 0.5
+```
+
+`--vae-tile-size` accepts one size or `WIDTHxHEIGHT`. A zero dimension uses the
+256-pixel default. Sizes are rounded down to a multiple of the VAE scale factor
+and capped at the current input dimensions. Explicit sizes below four latent
+pixels per axis (or the full axis when it is smaller) are rejected. Encoding and
+decoding use the same spatial sizes, without an additional encoding multiplier.
+Inputs that fit within a tile are processed as one tile.
+
+For a 512x512 image with the default 50% overlap, both encoding and decoding use
+3x3 tiles. A 256-pixel tile corresponds to 32 latent pixels for an 8x VAE, 16 for
+a 16x VAE, and 8 for a 32x VAE. Smaller tiles reduce each graph's memory demand,
+but overlapping work can increase processing time and tiling can affect image
+quality, especially during encoding. Use larger tiles when more context is needed.
+
+`--vae-relative-tile-size` overrides the absolute size on each axis with a positive
+value. Values up to and including 1 specify a fraction of the current input size;
+values greater than 1 specify a target number of tiles per axis, accounting for
+overlap. For example, `0.5x0.5` uses half the width and height in both encode and
+decode. The target overlap is clamped to 0 through 0.5 and the actual overlap is
+adjusted to fit the image. On an axis that does not wrap, a tile size that would leave
+neighboring tiles overlapping by less than half the target (or by fewer than 2 latent
+pixels) is replaced. The replacement is the size that decodes the fewest latent pixels
+along the axis (tiles times size) among sizes whose overlaps reach the full target overlap,
+capped at 8 latent pixels, preferring sizes within 25% above the request. It never uses
+more tiles than the nearest size that overlaps by half the target (smaller first, larger
+only up to twice the request), and falls back to that size when nothing better qualifies.
+Without it, a tile a little over half of the axis gives two tiles that barely overlap and
+the tile edge shows as a line. Size and overlap options
+require `--vae-tiling`.
+
+**Migration:** `--vae-tile-size` and the C/JSON fields `tile_size_w` and
+`tile_size_h` now use image pixels instead of latent units. The C/JSON fields
+`tile_size_x/y` have been renamed to `tile_size_w/h`, and `rel_size_x/y` to
+`rel_size_w/h`. The command-line option names are unchanged. For example, an old
+decode tile size of 32 corresponds to 256 pixels for an 8x VAE or 512 pixels for a
+16x VAE. Encoding no longer enlarges explicit or relative tile sizes.
+
+The main VAE decode and encode paths retry allocation failures with smaller tiles,
+even without `--vae-tiling`. Supported video VAEs first try temporal tiling; spatial
+retries use at most 256-pixel tiles initially and then halve the effective tile
+dimensions down to the minimum size. Each spatial retry must reduce the effective
+tile size. These runtime adjustments do not change the caller's parameters.
+Execution failures are not retried. Encoding retries the same way, without the
+temporal step.
+
+`--temporal-tiling` remains independent of spatial tiling. MiniMax H3 always uses
+spatial tiling (256x256 pixels and 25% overlap by default) and its own temporal
+windows. With `--vae-tiling`, its overlap follows `--vae-tile-overlap`; explicit
+spatial sizes are honored.
+
 ## Offload weights to the CPU to save VRAM without reducing generation speed.
 
 Using `--offload-to-cpu` allows you to offload weights to the CPU, saving VRAM without reducing generation speed.
@@ -63,6 +149,12 @@ See [backend selection](./backend.md) for full syntax.
 
 When a graph has cut markers and its missing weights plus incremental compute workspace exceed the available device headroom, it runs its fixed segment list in order. A reusable monolithic compute buffer is not counted as a new allocation. An explicit `--max-vram` budget deducts already-resident managed weights and compute/cache buffers registered by every runner sharing the device, so later graph runs remain segmented when the full graph exceeds the budget. The current segment's weights are pinned during compute, and the next parameter-bearing segment is prefetched when the device supports asynchronous transfer. No opt-in streaming flag is required.
 
+When choosing between monolithic and segmented execution, the runner requires
+an additional 128 MiB of headroom in both available device memory and any explicit
+managed budget. This planning headroom absorbs small allocation estimate changes;
+subsequent capacity checks can consume it while still preserving the 512 MiB device
+scratch reserve and respecting the managed budget.
+
 - `--max-vram <GiB>` optionally lowers the live-memory limit. A positive value is a managed per-device budget, `0` uses the device's current free memory without an explicit budget, and a negative value snapshots free memory at startup while reserving that many GiB (`--max-vram -1` reserves about 1 GiB). Driver contexts and unrelated external allocations remain outside the managed budget.
 - `--disable-prefetch` disables asynchronous next-segment prefetch while retaining synchronous loading, eviction, and segmented execution.
 - `--disable-segmented-compute` forces monolithic graph execution for diagnostics or compatibility, even when the automatic memory check would select segments.
@@ -92,3 +184,17 @@ Use `--params-backend diffusion=disk` instead when reducing system RAM residency
 ## Use quantization to reduce memory usage.
 
 [quantization](./quantization_and_gguf.md)
+
+## Fused DiT and VAE kernels (CUDA)
+
+With the fork's ggml patches (`scripts/unsloth/ggml-patches/0007-ggml-cuda-generic-dit-vae.patch`) these run by
+default on CUDA. Each gives the same output bit for bit as the ops it replaces, and each has a switch (`=0` turns it off):
+
+| what | families | switch |
+|---|---|---|
+| Shared DiT attention (`Rope::attention`, flash attention on): RoPE, head-major relayout, kv scale and F16 cast of q / k / v in one op per tensor | FLUX.1, FLUX.2, Chroma, Z-Image, Qwen-Image, Wan, and the other DiTs using `Rope::attention` | `SD_FUSED_ROPE_ATTN` |
+| GroupNorm spread over the whole GPU, fused with the per-channel affine and the SiLU after it | SD 1.x / SDXL U-Net, every KL VAE decode (SD, SDXL, FLUX, Z-Image, Chroma) | `GGML_CUDA_GROUP_NORM_PAR` |
+| Channel RMS norm of the Wan-style VAE computed in place, without the two relayout copies around it (large layers only, so mostly untiled decodes; `=2` forces it at every size) | Qwen-Image, Qwen-Image-2.1, Wan VAE | `GGML_CUDA_NORM_CHANNEL` |
+| `x*s + y (+ z)` elementwise chains (adaLN modulate, gated residual) in one pass | most DiTs | `GGML_CUDA_FUSE_MUL_ADD` |
+
+Convolutions on a batch of one (and the Wan VAE's single-frame 2D convolutions) also skip relayout copies that move no data.
diff --git a/docs/pid.md b/docs/pid.md
index 88adb0e2f6..3737f32ffe 100644
--- a/docs/pid.md
+++ b/docs/pid.md
@@ -27,7 +27,7 @@ PiD and PiD 1.5 require an external Gemma 2 `tokenizer.json` matching the text e
 ## Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\pid_flux1_512_to_2048_4step_bf16.safetensors --llm "..\models\text_encoders\gemma_2_2b_it_elm_bf16.safetensors" --tokenizer ..\models\tokenizers\tokenizer_gemma2.json --vae ..\models\vae\ae.sft --vae-format flux --cfg-scale 1.0  -p "a lovely cat" -r ..\assets\ernie_image\turbo_example.png --diffusion-fa -v --steps 4 -H 2048 -W 2048 --rng cpu
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\pid_flux1_512_to_2048_4step_bf16.safetensors --llm "..\models\text_encoders\gemma_2_2b_it_elm_bf16.safetensors" --tokenizer ..\models\tokenizers\tokenizer_gemma2.json --vae ..\models\vae\ae.sft --vae-format flux --cfg-scale 1.0  -p "a lovely cat" -r ..\assets\ernie_image\turbo_example.png -v --steps 4 -H 2048 -W 2048 --rng cpu
 ```
 
 Before:
diff --git a/docs/pixart.md b/docs/pixart.md
new file mode 100644
index 0000000000..f9386abf16
--- /dev/null
+++ b/docs/pixart.md
@@ -0,0 +1,46 @@
+# How to Use
+
+You can run PixArt-α / PixArt-Σ with stable-diffusion.cpp.
+
+PixArt is a DiT-based text-to-image model family conditioned by a T5-XXL text
+encoder and a 4-channel VAE: SDXL-style for PixArt-Σ and SD1.x-style for PixArt-α.
+
+## Download weights
+
+- Download the transformer (diffusion model)
+    - PixArt-Σ XL-2 1024-MS: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/transformer
+    - PixArt-α XL-2 1024-MS: https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS/tree/main/transformer
+- Download the T5-XXL text encoder
+    - safetensors: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/text_encoder
+- Download the VAE
+    - PixArt-Σ: https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS/tree/main/vae
+    - PixArt-α: https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS/tree/main/vae
+    - Use the VAE matching the checkpoint's latent space. For TAE decoding or
+      preview, use TAESDXL for PixArt-Σ and TAESD for PixArt-α.
+- Tokenizer: the T5 vocabulary is embedded; no extra tokenizer file is needed.
+
+## Examples
+
+```
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\pixart_sigma_xl2_1024_ms.safetensors --t5xxl ..\models\text_encoders\t5xxl.safetensors --vae ..\models\vae\pixart_vae.safetensors -p "a lovely cat" --cfg-scale 4.5 -W 1024 -H 1024 --steps 20 -v
+```
+
+## Notes
+
+- The VAE scaling factor defaults to `0.13025` for PixArt-Σ. PixArt-α
+  checkpoints with resolution micro-condition weights use `0.18215`.
+  PixArt-α 512 has the same tensor layout as PixArt-Σ, so it requires an
+  explicit override: `--model-args "pixart_vae_scale_factor=0.18215"`.
+  This argument can also override the scale for other compatible checkpoints.
+- PixArt-Σ checkpoints compute 2D sincos positional embeddings at runtime;
+  the trained grid is 64x64 patches with an interpolation scale of 2.
+  For checkpoints trained at a different resolution, the positional embedding
+  parameters can be adjusted via model args:
+  `--model-args "pixart_pos_embed_base_size=<trained grid>,pixart_interpolation_scale=<scale>"`
+  (e.g. `pixart_pos_embed_base_size=32,pixart_interpolation_scale=1,pixart_vae_scale_factor=0.18215` for
+  PixArt-α XL-2 512).
+- Checkpoints carrying resolution/aspect-ratio micro-condition weights are
+  detected but those conditions are not applied yet; a warning is logged and
+  generation proceeds with the timestep embedding only.
+- The transformer predicts 8 channels (noise + learned variance); only the
+  noise half is used for sampling, matching the reference implementation.
diff --git a/docs/qwen_image.md b/docs/qwen_image.md
index fdadaef90a..00ee50e24b 100644
--- a/docs/qwen_image.md
+++ b/docs/qwen_image.md
@@ -14,7 +14,7 @@
 ## Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\qwen-image-Q8_0.gguf --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf  -p '一个穿着"QWEN"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线： 探索视觉生成基础模型的极限，开创理解与生成一体化的未来。二、Qwen-Image的模型特色：1、复杂文字渲染。支持中英渲染、自动布局； 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景：赋能专业内容创作、助力生成式AI发展。”' --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu -H 1024 -W 1024 --diffusion-fa --flow-shift 3
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\qwen-image-Q8_0.gguf --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf  -p '一个穿着"QWEN"标志的T恤的中国美女正拿着黑色的马克笔面相镜头微笑。她身后的玻璃板上手写体写着 “一、Qwen-Image的技术路线： 探索视觉生成基础模型的极限，开创理解与生成一体化的未来。二、Qwen-Image的模型特色：1、复杂文字渲染。支持中英渲染、自动布局； 2、精准图像编辑。支持文字编辑、物体增减、风格变换。三、Qwen-Image的未来愿景：赋能专业内容创作、助力生成式AI发展。”' --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu -H 1024 -W 1024 --flow-shift 3
 ```
 
 <img alt="qwen example" src="../assets/qwen/example.png" />
diff --git a/docs/qwen_image_2.1.md b/docs/qwen_image_2.1.md
index 08a50875ec..8663731465 100644
--- a/docs/qwen_image_2.1.md
+++ b/docs/qwen_image_2.1.md
@@ -16,6 +16,8 @@ Qwen Image 2.1 supports text-to-image generation and image editing, using Qwen3-
 
 Use `qwen_image_2.1_vae_bf16.safetensors` with this model. The earlier Qwen Image and Wan 2.2 VAE weights are not interchangeable with the Qwen Image 2.1 VAE weights.
 
+For faster encoding, decoding, or previews, see [TAEQI2.1](taesd.md#qwen-image-21-taeqi21).
+
 ## Examples
 
 Run the following commands from the build directory. Use image dimensions divisible by 32. The resolution-dependent flow schedule is selected automatically.
@@ -30,6 +32,16 @@ Run the following commands from the build directory. Use image dimensions divisi
 
 To use GGUF diffusion weights, set `--diffusion-model` to the path of a file such as `qwen_image_2.1-Q4_K.gguf`.
 
+### Turbo
+
+For [Qwen Image 2.1 Turbo](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), use the Turbo diffusion weights and select its fixed 8-step sigma schedule with `--sigmas qwen-image-2.1-turbo`. For example, add these sampling options to the command above:
+
+```text
+--sigmas qwen-image-2.1-turbo --steps 8 --cfg-scale 1.0 --sampling-method euler
+```
+
+The preset is equivalent to `--sigmas 1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0`. It only supplies the sigma values; CFG scale and sampling method remain independently configurable. As with an explicit sigma list, the sigma count determines the actual number of sampling steps. The same preset name is also accepted by `--hires-sigmas` for the highres fix second pass.
+
 ### Image editing
 
 Pass the reference image with `-r` and describe the edit in `-p`. Vision weights are required; the example below loads them separately with `--llm_vision`.
@@ -39,3 +51,49 @@ Pass the reference image with `-r` and describe the edit in `-p`. Vision weights
 ```
 
 For multiple reference images, repeat `-r` in the desired order, for example `-r first.png -r second.png`.
+
+### Prefix cache
+
+By default, the first denoising call for each fixed condition saves the text and reference-image keys and values from every transformer layer. Later calls only compute the target-image tokens. Positive and negative conditions use separate caches, which are released when sampling ends.
+
+Set `qwen_image_2_1_prefix_cache_type` in `--model-args` to `auto` or a type name using the same parser and case-sensitive names as `--type`:
+
+- `auto` (default): use FP16 only when Flash Attention is enabled, Sage Attention is disabled, the attention scale is unchanged, and every attention operation in the cache-writing or cache-reading graph selects Flash Attention after backend support checks. If an operation falls back, rebuild the prefix in FP32 before executing and keep FP32 for the rest of that sampling run.
+- `f32`: always store FP32 keys and values.
+- `f16`: always store FP16 keys and values, including with ordinary attention or custom attention scaling. This saves cache memory but can introduce additional rounding error.
+- Other types, such as `bf16`, `q4_1`, `q5_0`, `q5_1`, `q8_0`, `q4_K`, `q6_K`, `iq4_nl`, and `iq4_xs`: use the requested storage type if the ggml build provides runtime conversion to and from FP32. Quantization is lossy and must be selected explicitly; `auto` never selects a quantized type.
+
+Cache data is packed into contiguous rows of `hidden_size` elements before conversion, so 256-element quantization blocks work with the model's 128-element attention heads without padding. The type's block size must divide `hidden_size`. Unknown types, types lacking runtime conversion (for example `q8_1` and several IQ formats), and incompatible block sizes are ignored with a warning, leaving the previous setting or the default `auto` unchanged.
+
+For example, use `--model-args qwen_image_2_1_prefix_cache_type=q8_0` to enable 8-bit cache storage. Cached keys and values are converted back to the attention input type before concatenating with the current target tokens. This reduces persistent cache memory; attention working buffers still use floating-point values, and conversion adds work on each step. Backends without the required conversion operations use the existing CPU fallback.
+
+For the default 32-layer model, a prefix of 4096 tokens takes approximately the following memory per condition, excluding weights, working buffers, and allocation overhead:
+
+| Cache type | Memory |
+| --- | ---: |
+| `f32` | 4 GiB |
+| `f16` | 2 GiB |
+| `q8_0` | 1.0625 GiB |
+| `q4_0` | 0.5625 GiB |
+
+The runner accounts for the cache when checking the memory budget. If a cached execution runs out of memory, it releases the prefix caches, disables caching for the rest of that sampling run, and retries the full sequence once. Per-step conditioning extensions currently use the full-sequence path.
+
+Disable this optimization with `--model-args qwen_image_2_1_prefix_cache=false`. It reuses step-independent activations; numerical results can still differ slightly because the matrix sizes change.
+
+### Alpha channel
+
+This model supports alpha channel output. As the model determines whether to output a regular image or with transparency through the prompt, according to [official recommendation](https://github.com/QwenLM/Qwen-Image-2.1#transparent-image-generation-rgba), use the following prompt format for better results:
+
+> `This is an RGBA image with transparency. <your description>. The image has alpha channel and the background is transparent.`
+
+Since transparency is decided by the prompt rather than by the input or an explicit switch, the same format applies equally to editing, whether or not the reference image itself has an alpha channel. Note that alpha is kept only in `.png` and `.webp` outputs; saving as `.jpg` drops the transparency.
+
+Here are some examples ran with Q6_K quantization:
+| Input | Prompt | Output |
+| --- | --- | --- |
+| ![Qwen Image 2.1 alpha input example 1](../assets/qwen/qwen-image-2.1-alpha-in1.png) | This is an RGBA image with transparency. Replace the text "BLOOM" with "Qwen Image 2.1", keeping the same font of the original text. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 1](../assets/qwen/qwen-image-2.1-alpha-out1.png) |
+| ![Qwen Image 2.1 alpha input example 2](../assets/logo.png) | This is an RGBA image with transparency. Remove the background of the image, keeping only the text and cat. The image has alpha channel and the background is transparent. | ![Qwen Image 2.1 alpha output example 2](../assets/qwen/qwen-image-2.1-alpha-out2.png) |
+
+### Other features
+
+Other features of the model could be found on the [model card from QwenLM/Qwen-Image-2.1 repo](https://github.com/QwenLM/Qwen-Image-2.1), including 2 finetuned prompt rewriting Qwen3.5-9B model.
diff --git a/docs/qwen_image_edit.md b/docs/qwen_image_edit.md
index 2f08850287..2d53d83354 100644
--- a/docs/qwen_image_edit.md
+++ b/docs/qwen_image_edit.md
@@ -23,7 +23,7 @@
 ### Qwen Image Edit
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\Qwen_Image_Edit-Q8_0.gguf --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --seed 1118877715456453
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\Qwen_Image_Edit-Q8_0.gguf --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --seed 1118877715456453
 ```
 
 <img alt="qwen_image_edit" src="../assets/qwen/qwen_image_edit.png" />
@@ -32,7 +32,7 @@
 ### Qwen Image Edit 2509
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\Qwen-Image-Edit-2509-Q4_K_S.gguf --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf --llm_vision ..\models\text_encoders\Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'Qwen Image Edit 2509'"
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\Qwen-Image-Edit-2509-Q4_K_S.gguf --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\Qwen2.5-VL-7B-Instruct-Q8_0.gguf --llm_vision ..\models\text_encoders\Qwen2.5-VL-7B-Instruct.mmproj-Q8_0.gguf --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'Qwen Image Edit 2509'"
 ```
 
 <img alt="qwen_image_edit_2509" src="../assets/qwen/qwen_image_edit_2509.png" />
@@ -42,7 +42,7 @@
 To use the new Qwen Image Edit 2511 mode, `--model-args qwen_image_zero_cond_t=true` must be enabled; otherwise, image editing quality will degrade significantly.
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\qwen-image-edit-2511-Q4_K_M.gguf --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --diffusion-fa --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --model-args qwen_image_zero_cond_t=true
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\qwen-image-edit-2511-Q4_K_M.gguf --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --cfg-scale 2.5 --sampling-method euler -v --offload-to-cpu --flow-shift 3 -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'edit.cpp'" --model-args qwen_image_zero_cond_t=true
 ```
 
 <img alt="qwen_image_edit_2509" src="../assets/qwen/qwen_image_edit_2511.png" />
diff --git a/docs/sage_attention.md b/docs/sage_attention.md
index c051f3b70a..46c854051e 100644
--- a/docs/sage_attention.md
+++ b/docs/sage_attention.md
@@ -37,12 +37,12 @@ rebuild with CUDA 12.8 or newer to use SageAttention2++.
 
 ## Use
 
-Replace `--diffusion-fa` with `--sage-attn` in an existing command. For example,
+Add `--sage-attn` to enable SageAttention. For example,
 from the build directory:
 
 ```powershell
 .\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，
-形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --video-frames 33 --sage-attn
+形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --offload-to-cpu --video-frames 33 --sage-attn
 ```
 
 SageAttention currently handles unmasked attention with head dimensions from
@@ -51,9 +51,7 @@ and multiple batches. Dimensions below 64 are zero-padded to 64; dimensions
 between 65 and 127 are zero-padded to 128. The original softmax scale is preserved,
 and the output is cropped back to the original dimension. Other attention
 operations fall back to FlashAttention when supported, then ordinary attention.
-SageAttention takes precedence in diffusion
-when combined with `--fa` or `--diffusion-fa`; `--fa` continues to control other
-modules. Existing attention scaling overrides remain effective.
+SageAttention takes precedence over FlashAttention in diffusion. Existing attention scaling overrides remain effective.
 
 Attention quantization changes numerical results. Compare image quality and
 end-to-end generation time using the same seed, dimensions, and sampling
diff --git a/docs/sefi_image.md b/docs/sefi_image.md
index c6a3587b04..9989e4f8bd 100644
--- a/docs/sefi_image.md
+++ b/docs/sefi_image.md
@@ -32,19 +32,19 @@ The dispatcher picks `alpha` from the filename (`turbo` substring => 1.0, otherw
 ### 1B / 2B turbo
 
 ```
-./build/bin/sd-cli --diffusion-model /path/to/sefi_1b_turbo.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_2b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 1.0 --steps 4 -W 1024 -H 1024 -s 42 --diffusion-fa --offload-to-cpu -o out.png
+./build/bin/sd-cli --diffusion-model /path/to/sefi_1b_turbo.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_2b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 1.0 --steps 4 -W 1024 -H 1024 -s 42 --offload-to-cpu -o out.png
 ```
 
 ### 1B / 2B base
 
 ```
-./build/bin/sd-cli --diffusion-model /path/to/sefi_1b_base.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_2b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 4.0 --steps 50 -W 1024 -H 1024 -s 42 --diffusion-fa --offload-to-cpu -o out.png
+./build/bin/sd-cli --diffusion-model /path/to/sefi_1b_base.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_2b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 4.0 --steps 50 -W 1024 -H 1024 -s 42 --offload-to-cpu -o out.png
 ```
 
 ### 5B (needs streaming on 12 GiB VRAM)
 
 ```
-./build/bin/sd-cli --diffusion-model /path/to/sefi_5b_turbo.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_4b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 1.0 --steps 4 -W 1024 -H 1024 -s 42 --diffusion-fa --max-vram 8 --offload-to-cpu -o out.png
+./build/bin/sd-cli --diffusion-model /path/to/sefi_5b_turbo.safetensors --vae /path/to/flux2_ae.safetensors --llm /path/to/qwen3_vl_4b.safetensors -p "a photograph of an orange tabby cat sitting on a couch" --cfg-scale 1.0 --steps 4 -W 1024 -H 1024 -s 42 --max-vram 8 --offload-to-cpu -o out.png
 ```
 
 <img alt="SeFi-Image 5B turbo example" src="../assets/sefi_image/example.png" />
diff --git a/docs/sensenova_u1.md b/docs/sensenova_u1.md
index fd19c4d38e..57b22a4ef1 100644
--- a/docs/sensenova_u1.md
+++ b/docs/sensenova_u1.md
@@ -28,7 +28,6 @@ and the tokenizer files.
   --seed 42 \
   --sampling-method euler \
   --rng cuda \
-  --fa \
   --output output.png
 ```
 
diff --git a/docs/taesd.md b/docs/taesd.md
index a41c64d43d..954086d180 100644
--- a/docs/taesd.md
+++ b/docs/taesd.md
@@ -37,3 +37,15 @@ sd.cpp also supports [TAEHV](https://github.com/madebyollin/taehv) (#937), which
   ```
 
 Then simply replace the `--vae xxx.safetensors` with `--tae xxx.safetensors` in the commands. If it still out of VRAM, add `--vae-conv-direct` to your command though might be slower.
+
+### Qwen Image 2.1 (TAEQI2.1)
+
+For Qwen Image 2.1, use [taeqi2_1](https://github.com/madebyollin/taesd), which supports 64-channel latents, 16x spatial scaling, and RGBA images.
+
+Download the official [safetensors weights](https://huggingface.co/madebyollin/taeqi2_1/blob/main/taeqi2_1.safetensors) directly; no conversion is needed:
+
+```bash
+curl -L -o taeqi2_1.safetensors https://huggingface.co/madebyollin/taeqi2_1/resolve/main/taeqi2_1.safetensors
+```
+
+Replace `--vae PATH` with `--taesd taeqi2_1.safetensors` in the [Qwen Image 2.1 examples](qwen_image_2.1.md) to use it for encoding and decoding. For previews only, keep `--vae PATH` and add `--taesd taeqi2_1.safetensors --taesd-preview-only --preview tae`.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 22007db2f2..27731e1cce 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -1,5 +1,14 @@
 # Troubleshooting
 
+## Video model used in image generation mode
+
+If generation reports that a model cannot be run with `generate_image()`, add
+`--mode vid_gen` to the CLI command. `--video-frames` alone does not select video
+mode. Video models require this mode even when generating a single frame.
+Library callers must use `generate_video()` for these models; use
+`sd_ctx_supports_image_generation()` and `sd_ctx_supports_video_generation()` to
+check the available generation modes.
+
 ## Completely black or white images or videos / NaNs
 
 Some ggml backends can encounter numerical overflow during inference, producing
@@ -31,8 +40,7 @@ the intended output scale:
 - `--linear-scale` scales Linear inputs before matrix multiplication and rescales
   the result.
 - `--attn-scale` scales attention keys and values (K/V). It takes effect only in
-  the Flash Attention path, where `--fa` or `--diffusion-fa` is enabled and the
-  backend supports it.
+  the Flash Attention path, when selected and supported by the backend.
 
 The two values can be set independently and apply across model components. The
 default `0` preserves each model's built-in settings; `1` explicitly disables the
diff --git a/docs/wan.md b/docs/wan.md
index c7cf1dd9e8..87a960f305 100644
--- a/docs/wan.md
+++ b/docs/wan.md
@@ -1,5 +1,7 @@
 # How to Use
 
+Wan models require `-M vid_gen`, including single-frame generation. `--video-frames` alone does not select video mode. Library callers must use `generate_video()` instead of `generate_image()`.
+
 ## Download weights
 
 - Download Wan
@@ -38,6 +40,9 @@
             - safetensors: https://huggingface.co/Comfy-Org/Wan_2.2_ComfyUI_Repackaged/tree/main/split_files/diffusion_models
             - gguf: https://huggingface.co/QuantStack/Wan2.2-S2V-14B-GGUF/tree/main
             - int8_convrot safetensors: https://huggingface.co/noctrex/Wan2.2-S2V-14B-int8_convrot
+        - Wan2.2 VACE-Fun A14B
+            - safetensors: https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B
+            - gguf: https://huggingface.co/QuantStack/Wan2.2-VACE-Fun-A14B-GGUF/tree/main
 - Download vae
     - wan_2.1_vae (for all the wan model except Wan2.2 TI2V 5B)
         - safetensors: https://huggingface.co/Comfy-Org/Wan_2.1_ComfyUI_repackaged/blob/main/split_files/vae/wan_2.1_vae.safetensors
@@ -62,7 +67,7 @@
 ### Wan2.1 T2V 1.3B
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1_t2v_1.3B_fp16.safetensors --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1_t2v_1.3B_fp16.safetensors --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.1_1.3B_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -70,7 +75,7 @@
 ### Wan2.1 T2V 14B
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-t2v-14b-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa  --offload-to-cpu --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-t2v-14b-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480  --offload-to-cpu --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.1_14B_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -80,7 +85,7 @@
 ### Wan2.1 I2V 14B
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-i2v-14b-480p-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\models\clip_vision\clip_vision_h.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-i2v-14b-480p-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\models\clip_vision\clip_vision_h.safetensors -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.1_14B_i2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -88,7 +93,7 @@
 ### Wan2.2 T2V A14B
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --offload-to-cpu --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_14B_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -96,7 +101,7 @@
 ### Wan2.2 I2V A14B
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --offload-to-cpu --video-frames 33 --offload-to-cpu -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_14B_i2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -109,7 +114,7 @@ wav2vec2 audio encoder. Wan2.2 S2V requires the wan_2.1 vae (16 channel), not
 the wan2.2 vae.
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.2_s2v-14B-Q8_0.gguf --audio-encoder ..\models\audio_encoders\wav2vec2_large_english_fp16.safetensors --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a person is talking" --cfg-scale 6.0 --steps 20 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --vae-tiling --video-frames 81 -i ..\assets\cat_with_sd_cpp_42.png --audio .\input\speech.wav --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.2_s2v-14B-Q8_0.gguf --audio-encoder ..\models\audio_encoders\wav2vec2_large_english_fp16.safetensors --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a person is talking" --cfg-scale 6.0 --steps 20 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --offload-to-cpu --vae-tiling --video-frames 81 -i ..\assets\cat_with_sd_cpp_42.png --audio .\input\speech.wav --flow-shift 3.0
 ```
 
 Notes:
@@ -146,7 +151,7 @@ Notes:
 ### Wan2.2 T2V A14B T2I
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --offload-to-cpu --flow-shift 3.0
 ```
 
 <img width="832" height="480" alt="Wan2 2_14B_t2i" src="../assets/wan/Wan2.2_14B_t2i.png" />
@@ -154,7 +159,7 @@ Notes:
 ### Wan2.2 T2V 14B with Lora
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat<lora:wan2.2_t2v_lightx2v_4steps_lora_v1.1_low_noise:1><lora:|high_noise|wan2.2_t2v_lightx2v_4steps_lora_v1.1_high_noise:1>" --cfg-scale 3.5 --sampling-method euler --steps 4 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 4 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --offload-to-cpu --lora-model-dir ..\models\loras --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-T2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat<lora:wan2.2_t2v_lightx2v_4steps_lora_v1.1_low_noise:1><lora:|high_noise|wan2.2_t2v_lightx2v_4steps_lora_v1.1_high_noise:1>" --cfg-scale 3.5 --sampling-method euler --steps 4 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 4 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --offload-to-cpu --lora-model-dir ..\models\loras --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_14B_t2v_lora.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -166,7 +171,7 @@ Notes:
 #### T2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\models\vae\wan2.2_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --offload-to-cpu --video-frames 33 --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\models\vae\wan2.2_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --offload-to-cpu --video-frames 33 --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_5B_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -174,7 +179,7 @@ Notes:
 #### I2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\models\vae\wan2.2_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --offload-to-cpu --video-frames 33 -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.2_ti2v_5B_fp16.safetensors --vae ..\models\vae\wan2.2_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --offload-to-cpu --video-frames 33 -i ..\assets\cat_with_sd_cpp_42.png --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_5B_i2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -182,7 +187,7 @@ Notes:
 ### Wan2.1 FLF2V 14B
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-flf2v-14b-720p-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\models\clip_vision\clip_vision_h.safetensors -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-flf2v-14b-720p-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --clip_vision ..\models\clip_vision\clip_vision_h.safetensors -p "glass flower blossom" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0
 ```
 
 
@@ -191,7 +196,7 @@ Notes:
 ### Wan2.2 FLF2V 14B
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -p "glass flower blossom" -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.2-I2V-A14B-LowNoise-Q8_0.gguf --high-noise-diffusion-model  ..\models\diffusion_models\Wan2.2-I2V-A14B-HighNoise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -p "glass flower blossom" -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --video-frames 33 --offload-to-cpu --init-img ..\..\ComfyUI\input\start_image.png --end-img ..\..\ComfyUI\input\end_image.png --flow-shift 3.0
 ```
 
 <video src=../assets/wan/Wan2.2_14B_flf2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -201,7 +206,7 @@ Notes:
 #### T2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 1 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --video-frames 1 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_1.3B_vace_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -210,7 +215,7 @@ Notes:
 #### R2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_1.3B_vace_r2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -221,7 +226,7 @@ Notes:
 ```
 mkdir post+depth
 ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\frame_%04d.jpg
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\wan2.1-vace-1.3b-q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_1.3B_vace_v2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -231,7 +236,7 @@ ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\fr
 #### T2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_14B_vace_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -240,7 +245,7 @@ ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\fr
 #### R2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 --diffusion-fa -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_14B_vace_r2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
@@ -250,7 +255,29 @@ ffmpeg -i ..\..\ComfyUI\input\post+depth.mp4 -qscale:v 1 -vf fps=8 post+depth\fr
 #### V2V
 
 ```
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 --diffusion-fa -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\Wan2.1_14B_VACE-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf  -p "The girl is dancing in a sea of flowers, slowly moving her hands. There is a close - up shot of her upper body. The character is surrounded by other transparent glass flowers in the style of Nicoletta Ceccoli, creating a beautiful, surreal, and emotionally expressive movie scene with a white. transparent feel and a dreamyl atmosphere." --cfg-scale 6.0 --sampling-method euler -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 480 -H 832 -i ..\..\ComfyUI\input\dance_girl.jpg --control-video ./post+depth --video-frames 33 --offload-to-cpu
 ```
 
 <video src=../assets/wan/Wan2.1_14B_vace_v2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
+
+### Wan2.2 VACE-Fun A14B
+
+VACE-Fun runs as a MoE pair: `--diffusion-model` takes the low-noise expert and
+`--high-noise-diffusion-model` the high-noise one. Reference-to-video uses `-i`
+for the reference image, same as Wan2.1 VACE.
+
+#### R2V
+
+```
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-low-noise-Q8_0.gguf --high-noise-diffusion-model ..\models\diffusion_models\Wan2.2-VACE-Fun-A14B-high-noise-Q8_0.gguf --vae ..\models\vae\wan_2.1_vae.safetensors --t5xxl ..\models\text_encoders\umt5-xxl-encoder-Q8_0.gguf -p "a lovely cat" --cfg-scale 3.5 --sampling-method euler --steps 10 --high-noise-cfg-scale 3.5 --high-noise-sampling-method euler --high-noise-steps 8 -v -n "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部， 畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" -W 832 -H 480 -i ..\assets\cat_with_sd_cpp_42.png --video-frames 33 --offload-to-cpu
+```
+
+<video src=../assets/wan/Wan2.2_A14B_vace_r2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>
+
+#### T2V
+
+Same command without `-i` (VACE context is synthesized from an empty control
+video, like Wan2.1 VACE t2v).
+
+> On GPUs with ~12 GB VRAM, VACE also needs `--vae-tiling` — the control-video
+> encode can exceed the budget otherwise.
diff --git a/docs/z_image.md b/docs/z_image.md
index 5ad98a1e1c..8dca31959b 100644
--- a/docs/z_image.md
+++ b/docs/z_image.md
@@ -21,7 +21,7 @@ You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or ev
 ### Z-Image-Turbo
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  z_image_turbo-Q3_K.gguf --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 1.0 -v --offload-to-cpu --diffusion-fa -H 1024 -W 512 --steps 8
+.\bin\Release\sd-cli.exe --diffusion-model  z_image_turbo-Q3_K.gguf --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\Qwen3-4B-Instruct-2507-Q4_K_M.gguf -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 1.0 -v --offload-to-cpu -H 1024 -W 512 --steps 8
 ```
 
 <img width="256" alt="z-image example" src="../assets/z_image/q3_K.png" />
@@ -29,7 +29,7 @@ You can run Z-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — or ev
 ### Z-Image-Base
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\z_image_bf16.safetensors --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa -H 1024 -W 512
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\z_image_bf16.safetensors --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "A cinematic, melancholic photograph of a solitary hooded figure walking through a sprawling, rain-slicked metropolis at night. The city lights are a chaotic blur of neon orange and cool blue, reflecting on the wet asphalt. The scene evokes a sense of being a single component in a vast machine. Superimposed over the image in a sleek, modern, slightly glitched font is the philosophical quote: 'THE CITY IS A CIRCUIT BOARD, AND I AM A BROKEN TRANSISTOR.' -- moody, atmospheric, profound, dark academic" --cfg-scale 5.0 -v --offload-to-cpu -H 1024 -W 512
 ```
 
 <img width="256" alt="z-image example" src="../assets/z_image/base_bf16.png" />
diff --git a/docs/z_image_l2p.md b/docs/z_image_l2p.md
new file mode 100644
index 0000000000..a7836f54df
--- /dev/null
+++ b/docs/z_image_l2p.md
@@ -0,0 +1,21 @@
+# Z-Image L2P
+
+[L2P](https://nju-pcalab.github.io/projects/L2P/) (Latent-to-Pixel) transfers Z-Image-Turbo to pixel space. The VAE is replaced by 16x16 patch tokenization on RGB pixels and a small convolutional local decoder that turns the last DiT hidden states back into pixels, so no VAE is needed. The text encoder is the Qwen3-4B used by Z-Image-Turbo.
+
+## Download weights
+
+- Download L2P (1K)
+    - safetensors: https://huggingface.co/zhen-nan/L2P/tree/main
+- Download Qwen3 4b
+    - safetensors: https://huggingface.co/Comfy-Org/z_image_turbo/tree/main/split_files/text_encoders
+    - gguf: https://huggingface.co/unsloth/Qwen3-4B-Instruct-2507-GGUF/tree/main
+
+## Text-to-image
+
+Do not pass `--vae`. Image dimensions must be multiples of 16. The reference pipeline uses 30 steps with a CFG scale of 2.0.
+
+The released checkpoint keeps part of the transformer in F32. Pass `--type bf16` to load it in BF16, as the reference pipeline runs it; this reduces the diffusion model from about 18.6 GB to 11.8 GB.
+
+```bash
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\model-1k-merge.safetensors --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "an origami pig on fire in the middle of a dark room with a pentagram on the floor" --cfg-scale 2.0 --steps 30 --sampling-method euler -W 1024 -H 1024 --type bf16 -v -o z_image_l2p.png
+```
diff --git a/examples/cli/README.md b/examples/cli/README.md
index fe47122b63..bf9de1862d 100644
--- a/examples/cli/README.md
+++ b/examples/cli/README.md
@@ -11,9 +11,26 @@ Logging defaults to `info`. Use `--log-level <level>` to select `debug`, `verbos
 messages at that level and all less detailed levels. `-v` and `--verbose` are
 equivalent to `--log-level verbose`. If repeated, the last logging option wins.
 
+`--flash-attn on|off|auto` defaults to `auto`, selecting flash attention for each
+compute backend during context creation. See [Backend selection](../../docs/backend.md#flash-attention---flash-attn-onoffauto)
+for probing behavior.
+Use module assignments such as `--flash-attn auto,te=off,vae=on` to override
+individual modules.
+
+`--conv-direct on|off|auto` also defaults to `auto`, enabling direct convolution
+on backends with implicit GEMM implementations. Use `--conv-direct auto,vae=off`
+for module overrides; supported modules are `diffusion`, `vae`, and `controlnet`.
+See [Direct convolution](../../docs/backend.md#direct-convolution---conv-direct-onoffauto).
+
 For direct image repair or automatic post-generation YOLOv8 detection followed by cropped inpainting, see
 [ADetailer](../../docs/adetailer.md).
 
+Use repeatable `--image-preprocess` rules to select resizing, cropping, padding,
+and resampling separately for each image input. Add `canny=true` to any input
+rule for edge detection. See
+[Image preprocessing](../../docs/image_preprocessing.md) for input selectors,
+input defaults, downstream model processing, mask alignment, and examples.
+
 Metadata mode inspects PNG/JPEG container metadata without loading any model:
 
 ```bash
diff --git a/examples/cli/image_metadata.cpp b/examples/cli/image_metadata.cpp
index 8dd339c883..ba0d6da230 100644
--- a/examples/cli/image_metadata.cpp
+++ b/examples/cli/image_metadata.cpp
@@ -4,6 +4,7 @@
 #include <cctype>
 #include <cstdint>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <iomanip>
 #include <iostream>
@@ -276,7 +277,7 @@ namespace {
     }
 
     bool read_file(const std::string& path, std::vector<uint8_t>& data, std::string& error) {
-        std::ifstream fin(path, std::ios::binary);
+        std::ifstream fin(std::filesystem::u8path(path), std::ios::binary);
         if (!fin) {
             error = "failed to open file: " + path;
             return false;
diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp
index eb96d9b5fe..ca8ce4e79e 100644
--- a/examples/cli/main.cpp
+++ b/examples/cli/main.cpp
@@ -41,7 +41,6 @@ struct SDCliParams {
     std::string metadata_format = "text";
 
     sd_log_level_t log_level = SD_LOG_INFO;
-    bool canny_preprocess    = false;
     bool convert_name        = false;
 
     preview_t preview_method = PREVIEW_NONE;
@@ -107,10 +106,6 @@ struct SDCliParams {
         };
 
         options.bool_options = {
-            {"",
-             "--canny",
-             "apply canny preprocessor (edge detection)",
-             true, &canny_preprocess},
             {"",
              "--convert-name",
              "convert tensor name (for convert mode)",
@@ -193,7 +188,7 @@ struct SDCliParams {
             if (++index >= argc) {
                 return -1;
             }
-            imatrix_in.push_back(argv[index]);
+            imatrix_in.push_back(argv_to_utf8(index, argv));
             return 1;
         };
 
@@ -268,7 +263,6 @@ struct SDCliParams {
             << "  metadata_format: \"" << metadata_format << "\",\n"
             << "  log_level: " << log_level_name(log_level) << ",\n"
             << "  color: " << (color ? "true" : "false") << ",\n"
-            << "  canny_preprocess: " << (canny_preprocess ? "true" : "false") << ",\n"
             << "  convert_name: " << (convert_name ? "true" : "false") << ",\n"
             << "  preview_method: " << previews_str[preview_method] << ",\n"
             << "  preview_interval: " << preview_interval << ",\n"
@@ -328,16 +322,15 @@ void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
 
 bool load_images_from_dir(const std::string dir,
                           std::vector<SDImageOwner>& images,
-                          int expected_width  = 0,
-                          int expected_height = 0,
-                          int max_image_num   = 0) {
-    if (!fs::exists(dir) || !fs::is_directory(dir)) {
+                          int max_image_num = 0) {
+    const fs::path dir_path = fs::u8path(dir);
+    if (!fs::exists(dir_path) || !fs::is_directory(dir_path)) {
         LOG_ERROR("'%s' is not a valid directory\n", dir.c_str());
         return false;
     }
 
     std::vector<fs::directory_entry> entries;
-    for (const auto& entry : fs::directory_iterator(dir)) {
+    for (const auto& entry : fs::directory_iterator(dir_path)) {
         if (entry.is_regular_file()) {
             entries.push_back(entry);
         }
@@ -345,19 +338,20 @@ bool load_images_from_dir(const std::string dir,
 
     std::sort(entries.begin(), entries.end(),
               [](const fs::directory_entry& a, const fs::directory_entry& b) {
-                  return a.path().filename().string() < b.path().filename().string();
+                  return a.path().filename().u8string() < b.path().filename().u8string();
               });
 
     for (const auto& entry : entries) {
-        std::string path = entry.path().string();
-        std::string ext  = entry.path().extension().string();
+        std::string path = entry.path().u8string();
+        std::string ext  = entry.path().extension().u8string();
         std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
 
         if (ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".bmp" || ext == ".webp") {
             LOG_VERBOSE("load image %zu from '%s'", images.size(), path.c_str());
             int width             = 0;
             int height            = 0;
-            uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, expected_width, expected_height);
+            int loaded_channel    = 0;
+            uint8_t* image_buffer = load_image_from_file(path.c_str(), width, height, loaded_channel, 0, 0);
             if (image_buffer == nullptr) {
                 LOG_ERROR("load image from '%s' failed", path.c_str());
                 return false;
@@ -365,7 +359,7 @@ bool load_images_from_dir(const std::string dir,
 
             images.emplace_back(sd_image_t{(uint32_t)width,
                                            (uint32_t)height,
-                                           3,
+                                           (uint32_t)loaded_channel,
                                            image_buffer});
 
             if (max_image_num > 0 && static_cast<int>(images.size()) >= max_image_num) {
@@ -404,19 +398,19 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy,
     // is_noisy is set to true if the preview corresponds to noisy latents, false if it's denoised latents
     // unused in this app, it will either be always noisy or always denoised here
     if (frame_count == 1) {
-        fs::path path = cli_params->preview_path;
-        if (encoded_image_format_from_path(path.string()) == EncodedImageFormat::UNKNOWN)
+        fs::path path = fs::u8path(cli_params->preview_path);
+        if (encoded_image_format_from_path(path.u8string()) == EncodedImageFormat::UNKNOWN)
             path += ".png";
-        if (std::regex_search(path.string(), format_specifier_regex))
-            path = fs::path(format_frame_idx(path.string(), continuous_preview_counter++));
-        if (!write_image_to_file(path.string(),
+        if (std::regex_search(path.u8string(), format_specifier_regex))
+            path = fs::u8path(format_frame_idx(path.u8string(), continuous_preview_counter++));
+        if (!write_image_to_file(path.u8string(),
                                  image->data,
                                  image->width,
                                  image->height,
                                  image->channel,
                                  "",
                                  cli_params->compression_quality)) {
-            LOG_ERROR("save preview image to '%s' failed", path.string().c_str());
+            LOG_ERROR("save preview image to '%s' failed", path.u8string().c_str());
         }
     } else {
         int fps = cli_params->preview_method == PREVIEW_PROJ ? cli_params->preview_fps / 4 : cli_params->preview_fps;
@@ -427,12 +421,12 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy,
 }
 
 static fs::path get_video_audio_sidecar_path(const SDCliParams& cli_params) {
-    fs::path out_path     = cli_params.output_path;
+    fs::path out_path     = fs::u8path(cli_params.output_path);
     fs::path base_path    = out_path;
     fs::path ext          = out_path.has_extension() ? out_path.extension() : fs::path{};
-    std::string ext_lower = ext.string();
+    std::string ext_lower = ext.u8string();
     std::transform(ext_lower.begin(), ext_lower.end(), ext_lower.begin(), ::tolower);
-    const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.string());
+    const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.u8string());
     if (!ext.empty()) {
         if (output_format == EncodedImageFormat::JPEG ||
             output_format == EncodedImageFormat::PNG ||
@@ -457,14 +451,14 @@ bool save_results(const SDCliParams& cli_params,
     }
 
     namespace fs      = std::filesystem;
-    fs::path out_path = cli_params.output_path;
+    fs::path out_path = fs::u8path(cli_params.output_path);
 
     if (!out_path.parent_path().empty()) {
         std::error_code ec;
         fs::create_directories(out_path.parent_path(), ec);
         if (ec) {
             LOG_ERROR("failed to create directory '%s': %s",
-                      out_path.parent_path().string().c_str(), ec.message().c_str());
+                      out_path.parent_path().u8string().c_str(), ec.message().c_str());
             return false;
         }
     }
@@ -472,9 +466,9 @@ bool save_results(const SDCliParams& cli_params,
     fs::path base_path = out_path;
     fs::path ext       = out_path.has_extension() ? out_path.extension() : fs::path{};
 
-    std::string ext_lower = ext.string();
+    std::string ext_lower = ext.u8string();
     std::transform(ext_lower.begin(), ext_lower.end(), ext_lower.begin(), ::tolower);
-    const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.string());
+    const EncodedImageFormat output_format = encoded_image_format_from_path(out_path.u8string());
     if (!ext.empty()) {
         if (output_format == EncodedImageFormat::JPEG ||
             output_format == EncodedImageFormat::PNG ||
@@ -500,8 +494,8 @@ bool save_results(const SDCliParams& cli_params,
         std::string params          = gen_params.embed_image_metadata
                                           ? get_image_params(ctx_params, gen_params, metadata_seed, cli_params.mode)
                                           : "";
-        const bool ok               = write_image_to_file(path.string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
-        LOG_INFO("save result image %d to '%s' (%s)", idx, path.string().c_str(), ok ? "success" : "failure");
+        const bool ok               = write_image_to_file(path.u8string(), img.data, img.width, img.height, img.channel, params, cli_params.compression_quality);
+        LOG_INFO("save result image %d to '%s' (%s)", idx, path.u8string().c_str(), ok ? "success" : "failure");
         return ok;
     };
 
@@ -509,14 +503,14 @@ bool save_results(const SDCliParams& cli_params,
         if (generated_audio == nullptr) {
             return;
         }
-        if (write_wav_to_file(wav_path.string(),
+        if (write_wav_to_file(wav_path.u8string(),
                               generated_audio->data,
                               generated_audio->sample_count,
                               generated_audio->channels,
                               generated_audio->sample_rate)) {
-            LOG_INFO("save result audio to '%s'", wav_path.string().c_str());
+            LOG_INFO("save result audio to '%s'", wav_path.u8string().c_str());
         } else {
-            LOG_WARN("failed to save result audio to '%s'", wav_path.string().c_str());
+            LOG_WARN("failed to save result audio to '%s'", wav_path.u8string().c_str());
         }
     };
 
@@ -529,7 +523,7 @@ bool save_results(const SDCliParams& cli_params,
         pattern += ext;
 
         for (int i = 0; i < num_results; ++i) {
-            fs::path img_path = format_frame_idx(pattern.string(), output_begin_idx + i);
+            fs::path img_path = fs::u8path(format_frame_idx(pattern.u8string(), output_begin_idx + i));
             if (write_image(img_path, i)) {
                 sucessful_reults++;
             }
@@ -547,11 +541,11 @@ bool save_results(const SDCliParams& cli_params,
 
         fs::path video_path = base_path;
         video_path += ext;
-        std::string final_ext_lower = ext.string();
+        std::string final_ext_lower = ext.u8string();
         std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
         const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
-        if (create_video_from_sd_images(video_path.string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
-            LOG_INFO("save result video to '%s'", video_path.string().c_str());
+        if (create_video_from_sd_images(video_path.u8string().c_str(), results, num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
+            LOG_INFO("save result video to '%s'", video_path.u8string().c_str());
             if (generated_audio != nullptr && !mux_audio) {
                 fs::path wav_path = video_path;
                 wav_path.replace_extension(".wav");
@@ -559,7 +553,7 @@ bool save_results(const SDCliParams& cli_params,
             }
             return true;
         } else {
-            LOG_ERROR("Failed to save result video to '%s'", video_path.string().c_str());
+            LOG_ERROR("Failed to save result video to '%s'", video_path.u8string().c_str());
             return false;
         }
     }
@@ -651,10 +645,11 @@ int main(int argc, const char* argv[]) {
 
     SDCliParams cli_params;
     SDContextParams ctx_params;
+    ctx_params.conditioning_cache_size = 0;
     SDGenerationParams gen_params;
 
-    parse_args(argc, argv, cli_params, ctx_params, gen_params);
     sd_set_log_callback(sd_log_cb, (void*)&cli_params);
+    parse_args(argc, argv, cli_params, ctx_params, gen_params);
 
     if (cli_params.mode == METADATA) {
         MetadataReadOptions options;
@@ -702,12 +697,12 @@ int main(int argc, const char* argv[]) {
 
     LOG_VERBOSE("version: %s", version_string().c_str());
     LOG_VERBOSE("%s", sd_get_system_info());
-    LOG_VERBOSE("%s", cli_params.to_string().c_str());
-    LOG_VERBOSE("%s", ctx_params.to_string().c_str());
-    LOG_VERBOSE("%s", gen_params.to_string().c_str());
+    LOG_DEBUG("%s", cli_params.to_string().c_str());
+    LOG_DEBUG("%s", ctx_params.to_string().c_str());
+    LOG_DEBUG("%s", gen_params.to_string().c_str());
 
     if (!cli_params.imatrix_out.empty()) {
-        if (fs::exists(cli_params.imatrix_out) &&
+        if (fs::exists(fs::u8path(cli_params.imatrix_out)) &&
             std::find(cli_params.imatrix_in.begin(), cli_params.imatrix_in.end(), cli_params.imatrix_out) == cli_params.imatrix_in.end()) {
             LOG_WARN("imatrix file '%s' already exists and will be overwritten", cli_params.imatrix_out.c_str());
         }
@@ -722,6 +717,15 @@ int main(int argc, const char* argv[]) {
     }
 
     if (cli_params.mode == CONVERT) {
+        std::vector<sd_lora_t> loras;
+        loras.reserve(gen_params.lora_map.size() + gen_params.high_noise_lora_map.size());
+        for (const auto& [path, multiplier] : gen_params.lora_map) {
+            loras.push_back({false, multiplier, path.c_str()});
+        }
+        for (const auto& [path, multiplier] : gen_params.high_noise_lora_map) {
+            loras.push_back({true, multiplier, path.c_str()});
+        }
+
         bool success = convert_with_components(ctx_params.model_path.c_str(),
                                                ctx_params.clip_l_path.c_str(),
                                                ctx_params.clip_g_path.c_str(),
@@ -732,7 +736,9 @@ int main(int argc, const char* argv[]) {
                                                ctx_params.wtype,
                                                ctx_params.tensor_type_rules.c_str(),
                                                cli_params.convert_name,
-                                               ctx_params.n_threads);
+                                               ctx_params.n_threads,
+                                               loras.empty() ? nullptr : loras.data(),
+                                               (int)loras.size());
         if (!success) {
             LOG_ERROR("convert '%s'/'%s' to '%s' failed",
                       ctx_params.model_path.c_str(),
@@ -750,16 +756,8 @@ int main(int argc, const char* argv[]) {
 
     auto load_image_and_update_size = [&](const std::string& path,
                                           SDImageOwner& image,
-                                          bool resize_image    = true,
                                           int expected_channel = 3) -> bool {
-        int expected_width  = 0;
-        int expected_height = 0;
-        if (resize_image && gen_params.width_and_height_are_set()) {
-            expected_width  = gen_params.width;
-            expected_height = gen_params.height;
-        }
-
-        if (!load_sd_image_from_file(image.put(), path.c_str(), expected_width, expected_height, expected_channel)) {
+        if (!load_sd_image_from_file(image.put(), path.c_str(), 0, 0, expected_channel)) {
             LOG_ERROR("load image from '%s' failed", path.c_str());
             return false;
         }
@@ -781,7 +779,8 @@ int main(int argc, const char* argv[]) {
     };
 
     if (gen_params.init_image_path.size() > 0) {
-        if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) {
+        const bool native_init = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER;
+        if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image, native_init ? 0 : 3)) {
             return 1;
         }
     }
@@ -795,8 +794,8 @@ int main(int argc, const char* argv[]) {
     if (gen_params.ref_image_paths.size() > 0) {
         gen_params.ref_images.clear();
         for (auto& path : gen_params.ref_image_paths) {
-            SDImageOwner ref_image({0, 0, 3, nullptr});
-            if (!load_image_and_update_size(path, ref_image, false)) {
+            SDImageOwner ref_image({0, 0, 0, nullptr});
+            if (!load_image_and_update_size(path, ref_image, 0)) {
                 return 1;
             }
             gen_params.ref_images.push_back(std::move(ref_image));
@@ -837,41 +836,22 @@ int main(int argc, const char* argv[]) {
     if (gen_params.mask_image_path.size() > 0) {
         if (!load_sd_image_from_file(gen_params.mask_image.put(),
                                      gen_params.mask_image_path.c_str(),
-                                     gen_params.get_resolved_width(),
-                                     gen_params.get_resolved_height(),
+                                     0,
+                                     0,
                                      1)) {
             LOG_ERROR("load image from '%s' failed", gen_params.mask_image_path.c_str());
             return 1;
         }
-    } else {
-        sd_image_t generated_mask = {0, 0, 1, nullptr};
-        generated_mask.data       = (uint8_t*)malloc(gen_params.get_resolved_width() * gen_params.get_resolved_height());
-        if (generated_mask.data == nullptr) {
-            LOG_ERROR("malloc mask image failed");
-            return 1;
-        }
-        generated_mask.width  = gen_params.get_resolved_width();
-        generated_mask.height = gen_params.get_resolved_height();
-        memset(generated_mask.data, 255, gen_params.get_resolved_width() * gen_params.get_resolved_height());
-        gen_params.mask_image.reset(generated_mask);
     }
 
     if (gen_params.control_image_path.size() > 0) {
         if (!load_sd_image_from_file(gen_params.control_image.put(),
                                      gen_params.control_image_path.c_str(),
-                                     gen_params.get_resolved_width(),
-                                     gen_params.get_resolved_height())) {
+                                     0,
+                                     0)) {
             LOG_ERROR("load image from '%s' failed", gen_params.control_image_path.c_str());
             return 1;
         }
-        if (cli_params.canny_preprocess) {  // apply preprocessor
-            preprocess_canny(gen_params.control_image.get(),
-                             0.08f,
-                             0.08f,
-                             0.8f,
-                             1.0f,
-                             false);
-        }
     }
 
     if (gen_params.ip_adapter_image_path.size() > 0) {
@@ -888,8 +868,6 @@ int main(int argc, const char* argv[]) {
         gen_params.control_frames.clear();
         if (!load_images_from_dir(gen_params.control_video_path,
                                   gen_params.control_frames,
-                                  gen_params.get_resolved_width(),
-                                  gen_params.get_resolved_height(),
                                   gen_params.video_frames)) {
             return 1;
         }
@@ -898,10 +876,7 @@ int main(int argc, const char* argv[]) {
     if (!gen_params.pm_id_images_dir.empty()) {
         gen_params.pm_id_images.clear();
         if (!load_images_from_dir(gen_params.pm_id_images_dir,
-                                  gen_params.pm_id_images,
-                                  0,
-                                  0,
-                                  0)) {
+                                  gen_params.pm_id_images)) {
             return 1;
         }
     }
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index afea3afa09..b9fb780fdb 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -75,7 +75,7 @@ static std::string utf16_to_utf8(const std::wstring& wstr) {
     return utf8;
 }
 
-static std::string argv_to_utf8(int index, const char** argv) {
+std::string argv_to_utf8(int index, const char** argv) {
     (void)argv;
     int argc;
     wchar_t** argv_w = CommandLineToArgvW(GetCommandLineW(), &argc);
@@ -91,7 +91,7 @@ static std::string argv_to_utf8(int index, const char** argv) {
 }
 
 #else  // Linux / macOS
-static std::string argv_to_utf8(int index, const char** argv) {
+std::string argv_to_utf8(int index, const char** argv) {
     return std::string(argv[index]);
 }
 
@@ -378,6 +378,23 @@ static int parse_scale_override(int argc, const char** argv, int index, float& s
     return 1;
 }
 
+static int parse_on_off_arg(int argc, const char** argv, int index, const char* option, bool& value) {
+    if (++index >= argc) {
+        LOG_ERROR("%s requires 'on' or 'off'", option);
+        return -1;
+    }
+    const std::string arg = argv[index];
+    if (arg == "on") {
+        value = true;
+    } else if (arg == "off") {
+        value = false;
+    } else {
+        LOG_ERROR("invalid %s value '%s'; expected 'on' or 'off'", option, argv[index]);
+        return -1;
+    }
+    return 1;
+}
+
 ArgOptions SDContextParams::get_options() {
     ArgOptions options;
     options.string_options = {
@@ -518,7 +535,8 @@ ArgOptions SDContextParams::get_options() {
         {"",
          "--model-args",
          "extra model args, key=value list. Supports chroma_use_dit_mask, chroma_use_t5_mask, "
-         "chroma_t5_mask_pad, qwen_image_zero_cond_t",
+         "chroma_t5_mask_pad, qwen_image_zero_cond_t, qwen_image_2_1_prefix_cache, "
+         "qwen_image_2_1_prefix_cache_type (auto or a type name from --type)",
          (int)',',
          &model_args},
         {"",
@@ -571,6 +589,10 @@ ArgOptions SDContextParams::get_options() {
          "number of threads to use during computation (default: -1). "
          "If threads <= 0, then threads will be set to the number of CPU physical cores",
          &n_threads},
+        {"",
+         "--conditioning-cache-size",
+         "maximum number of conditioning results cached per model context (default: " + std::to_string(conditioning_cache_size) + ", 0 disables caching)",
+         &conditioning_cache_size},
     };
 
     options.bool_options = {
@@ -610,43 +632,10 @@ ArgOptions SDContextParams::get_options() {
          "--vae-on-cpu",
          "deprecated; use --backend vae=cpu",
          true, &vae_on_cpu},
-        {"",
-         "--fa",
-         "use flash attention",
-         true, &flash_attn},
-        {"",
-         "--diffusion-fa",
-         "use flash attention in the diffusion model only",
-         true, &diffusion_flash_attn},
         {"",
          "--sage-attn",
          "use native CUDA SageAttention in the diffusion model, with flash/default attention fallback",
          true, &sage_attn},
-        {"",
-         "--diffusion-conv-direct",
-         "use ggml_conv2d_direct in the diffusion model",
-         true, &diffusion_conv_direct},
-        {"",
-         "--vae-conv-direct",
-         "use direct 2D and 3D convolutions in the vae model",
-         true, &vae_conv_direct},
-    };
-
-    auto on_auto_fit_arg = [&](int argc, const char** argv, int index) {
-        if (++index >= argc) {
-            LOG_ERROR("--auto-fit requires 'on' or 'off'");
-            return -1;
-        }
-        const std::string arg = argv[index];
-        if (arg == "on") {
-            auto_fit = true;
-        } else if (arg == "off") {
-            auto_fit = false;
-        } else {
-            LOG_ERROR("invalid --auto-fit value '%s'; expected 'on' or 'off'", argv[index]);
-            return -1;
-        }
-        return 1;
     };
 
     auto on_type_arg = [&](int argc, const char** argv, int index) {
@@ -728,16 +717,83 @@ ArgOptions SDContextParams::get_options() {
          }},
         {"",
          "--attn-scale",
-         "flash-attention K/V scale override (float, default: 0 = model default, 1 = no scaling); requires --fa or --diffusion-fa",
+         "flash-attention K/V scale override (float, default: 0 = model default, 1 = no scaling); applies when flash attention is selected",
          [this](int argc, const char** argv, int index) {
              return parse_scale_override(argc, argv, index, attn_scale);
          }},
+        {"",
+         "--diffusion-conv-direct",
+         "deprecated; use --conv-direct auto,diffusion=on,controlnet=on",
+         [this](int, const char**, int) {
+             LOG_WARN("--diffusion-conv-direct is deprecated; use --conv-direct auto,diffusion=on,controlnet=on instead.");
+             diffusion_conv_direct = true;
+             return 0;
+         }},
+        {"",
+         "--vae-conv-direct",
+         "deprecated; use --conv-direct auto,vae=on",
+         [this](int, const char**, int) {
+             LOG_WARN("--vae-conv-direct is deprecated; use --conv-direct auto,vae=on instead.");
+             vae_conv_direct = true;
+             return 0;
+         }},
+        {"",
+         "--conv-direct",
+         "on|off|auto or diffusion/vae/controlnet assignments (default: auto), e.g. auto,vae=off. Overrides legacy convolution flags",
+         [this](int argc, const char** argv, int index) {
+             if (++index >= argc || argv[index][0] == '\0') {
+                 LOG_ERROR("--conv-direct requires on, off, auto, or module assignments");
+                 return -1;
+             }
+             conv_direct_mode = argv[index];
+             return 1;
+         }},
+        {"",
+         "--flash-attn",
+         "on|off|auto or module assignments (default: auto), e.g. auto,te=off,vae=on. Last repeated option wins",
+         [this](int argc, const char** argv, int index) {
+             if (++index >= argc) {
+                 LOG_ERROR("--flash-attn requires on, off, or auto");
+                 return -1;
+             }
+             const std::string value = argv[index];
+             if (value.empty()) {
+                 LOG_ERROR("--flash-attn requires a nonempty value");
+                 return -1;
+             }
+             flash_attn_mode = value;
+             return 1;
+         }},
+        {"",
+         "--fa",
+         "deprecated; use --flash-attn on",
+         [this](int, const char**, int) {
+             LOG_WARN("--fa is deprecated; use --flash-attn on instead.");
+             flash_attn_mode = "on";
+             return 0;
+         }},
+        {"",
+         "--diffusion-fa",
+         "deprecated; use --flash-attn off,diffusion=on",
+         [this](int, const char**, int) {
+             LOG_WARN("--diffusion-fa is deprecated; use --flash-attn off,diffusion=on instead.");
+             flash_attn_mode = "off,diffusion=on";
+             return 0;
+         }},
         {"",
          "--auto-fit",
          "on|off (default: on). Preserve --backend (otherwise select one GPU) and place weights on the compute GPU, "
          "RAM, another GPU, or disk in that order, according to available memory (--max-vram limits GPU budgets). "
          "Disabled by explicit --params-backend; uses automatic graph segmentation when needed",
-         on_auto_fit_arg},
+         [this](int argc, const char** argv, int index) {
+             return parse_on_off_arg(argc, argv, index, "--auto-fit", auto_fit);
+         }},
+        {"",
+         "--batched-cfg",
+         "on|off (default: on). Run the conditional and unconditional CFG branches in one batched UNet forward when supported",
+         [this](int argc, const char** argv, int index) {
+             return parse_on_off_arg(argc, argv, index, "--batched-cfg", batched_cfg);
+         }},
         {"",
          "--type",
          "weight type (examples: f32, f16, q4_0, q4_1, q5_0, q5_1, q8_0, q2_K, q3_K, q4_K). "
@@ -783,16 +839,17 @@ ArgOptions SDContextParams::get_options() {
 void SDContextParams::build_embedding_map() {
     static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt", ".ckpt"};
 
-    if (!fs::exists(embedding_dir) || !fs::is_directory(embedding_dir)) {
+    const fs::path embedding_dir_path = fs::u8path(embedding_dir);
+    if (!fs::exists(embedding_dir_path) || !fs::is_directory(embedding_dir_path)) {
         return;
     }
 
-    for (auto& p : fs::directory_iterator(embedding_dir)) {
+    for (auto& p : fs::directory_iterator(embedding_dir_path)) {
         if (!p.is_regular_file())
             continue;
 
         auto path       = p.path();
-        std::string ext = path.extension().string();
+        std::string ext = path.extension().u8string();
 
         bool valid = false;
         for (auto& e : valid_ext) {
@@ -804,8 +861,8 @@ void SDContextParams::build_embedding_map() {
         if (!valid)
             continue;
 
-        std::string key   = path.stem().string();
-        std::string value = path.string();
+        std::string key   = path.stem().u8string();
+        std::string value = path.u8string();
 
         embedding_map[key] = value;
     }
@@ -822,6 +879,10 @@ bool SDContextParams::resolve(SDMode mode) {
 }
 
 bool SDContextParams::validate(SDMode mode) {
+    if (conditioning_cache_size < 0) {
+        LOG_ERROR("error: conditioning-cache-size must be non-negative");
+        return false;
+    }
     if (mode == CONVERT) {
         const bool has_convert_input = model_path.length() != 0 ||
                                        clip_l_path.length() != 0 ||
@@ -898,6 +959,7 @@ std::string SDContextParams::to_string() const {
     std::ostringstream oss;
     oss << "SDContextParams {\n"
         << "  n_threads: " << n_threads << ",\n"
+        << "  conditioning_cache_size: " << conditioning_cache_size << ",\n"
         << "  model_path: \"" << model_path << "\",\n"
         << "  clip_l_path: \"" << clip_l_path << "\",\n"
         << "  clip_g_path: \"" << clip_g_path << "\",\n"
@@ -930,6 +992,7 @@ std::string SDContextParams::to_string() const {
         << "  max_vram: \"" << max_vram << "\",\n"
         << "  disable_prefetch: " << (disable_prefetch ? "true" : "false") << ",\n"
         << "  disable_segmented_compute: " << (disable_segmented_compute ? "true" : "false") << ",\n"
+        << "  batched_cfg: " << (batched_cfg ? "true" : "false") << ",\n"
         << "  eager_load: " << (eager_load ? "true" : "false") << ",\n"
         << "  backend: \"" << backend << "\",\n"
         << "  params_backend: \"" << params_backend << "\",\n"
@@ -940,8 +1003,8 @@ std::string SDContextParams::to_string() const {
         << "  control_net_cpu: " << (control_net_cpu ? "true" : "false") << ",\n"
         << "  clip_on_cpu: " << (clip_on_cpu ? "true" : "false") << ",\n"
         << "  vae_on_cpu: " << (vae_on_cpu ? "true" : "false") << ",\n"
-        << "  flash_attn: " << (flash_attn ? "true" : "false") << ",\n"
-        << "  diffusion_flash_attn: " << (diffusion_flash_attn ? "true" : "false") << ",\n"
+        << "  flash_attn_mode: " << flash_attn_mode << ",\n"
+        << "  conv_direct_mode: " << (conv_direct_mode.empty() ? "auto (legacy flags honored)" : conv_direct_mode) << ",\n"
         << "  sage_attn: " << (sage_attn ? "true" : "false") << ",\n"
         << "  linear_scale: " << linear_scale << ",\n"
         << "  attn_scale: " << attn_scale << ",\n"
@@ -992,14 +1055,15 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
     sd_ctx_params.pulid_weights_path              = pulid_weights_path.c_str();
     sd_ctx_params.tensor_type_rules               = tensor_type_rules.c_str();
     sd_ctx_params.n_threads                       = n_threads;
+    sd_ctx_params.conditioning_cache_size         = conditioning_cache_size;
     sd_ctx_params.wtype                           = wtype;
     sd_ctx_params.rng_type                        = rng_type;
     sd_ctx_params.sampler_rng_type                = sampler_rng_type;
     sd_ctx_params.prediction                      = prediction;
     sd_ctx_params.lora_apply_mode                 = lora_apply_mode;
     sd_ctx_params.enable_mmap                     = enable_mmap;
-    sd_ctx_params.flash_attn                      = flash_attn;
-    sd_ctx_params.diffusion_flash_attn            = diffusion_flash_attn;
+    sd_ctx_params.flash_attn_mode                 = flash_attn_mode.c_str();
+    sd_ctx_params.conv_direct_mode                = conv_direct_mode.c_str();
     sd_ctx_params.sage_attn                       = sage_attn;
     sd_ctx_params.linear_scale                    = linear_scale;
     sd_ctx_params.attn_scale                      = attn_scale;
@@ -1011,6 +1075,7 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) {
     sd_ctx_params.max_vram                        = max_vram.c_str();
     sd_ctx_params.disable_prefetch                = disable_prefetch;
     sd_ctx_params.disable_segmented_compute       = disable_segmented_compute;
+    sd_ctx_params.batched_cfg                     = batched_cfg;
     sd_ctx_params.eager_load                      = eager_load;
     sd_ctx_params.backend                         = effective_backend.c_str();
     sd_ctx_params.params_backend                  = effective_params_backend.c_str();
@@ -1115,7 +1180,7 @@ ArgOptions SDGenerationParams::get_options() {
          &hires_upscaler},
         {"",
          "--extra-sample-args",
-         "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; llada_image supports uniform; lms supports lms_max_order, lms_shift, lms_divisions; noise-injecting samplers support noise_sampler with value iid (default except for dpm++2m_sde_bt) or brownian_tree; brownian_tree_rng supports cpu (default), cuda, std_default or sampler_rng",
+         "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; MOSS-Image supports moss_uncond_skip, moss_uncond_skip_start, moss_uncond_skip_end (sigma-percent window of its block-22 negative-branch skip); lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; llada_image supports uniform; lms supports lms_max_order, lms_shift, lms_divisions; noise-injecting samplers support noise_sampler with value iid (default except for dpm++2m_sde_bt) or brownian_tree; brownian_tree_rng supports cpu (default), cuda, std_default or sampler_rng",
          (int)',',
          &extra_sample_args},
         {"",
@@ -1128,6 +1193,9 @@ ArgOptions SDGenerationParams::get_options() {
          "Key-value list to set up the way the reference images are processed (empty = auto-detect from model weigths)",
          (int)',',
          &ref_image_args},
+        {"", "--image-preprocess",
+         "Image preprocessing rule: target=init|end|mask|control|ref|ip-adapter|id|control-frame,index=N,mode=auto|none|stretch|crop|crop-resize|fit-pad,filter=auto|nearest|nearest-exact|bilinear|bicubic|lanczos,antialias=auto|true|false,width=W,height=H,anchor=center|top|bottom|left|right,pad_color=#RRGGBB[AA],canny=true|false. Repeat for multiple rules.",
+         (int)';', &image_preprocess},
     };
 
     options.int_options = {
@@ -1308,11 +1376,6 @@ ArgOptions SDGenerationParams::get_options() {
          "automatically increase the indices of references images based on the order they are listed (starting with 1).",
          true,
          &increase_ref_index},
-        {"",
-         "--disable-auto-resize-ref-image",
-         "disable auto resize of ref images",
-         false,
-         &auto_resize_ref_image},
         {"",
          "--circular",
          "enable circular padding on both axes for tileable output",
@@ -1332,7 +1395,7 @@ ArgOptions SDGenerationParams::get_options() {
          &embed_image_metadata},
         {"",
          "--vae-tiling",
-         "process vae in tiles to reduce memory usage",
+         "process vae encode and decode in spatial tiles to reduce memory usage (default: 256x256 image pixels)",
          true,
          &vae_tiling_params.enabled},
         {"",
@@ -1456,6 +1519,9 @@ ArgOptions SDGenerationParams::get_options() {
             return -1;
         }
         std::string sigmas_str = value;
+        if (sigmas_str == "qwen-image-2.1-turbo") {
+            sigmas_str = "1.0,0.978453,0.95418,0.926626,0.89508,0.845148,0.704534,0.414568,0.0";
+        }
         if (!sigmas_str.empty() && sigmas_str.front() == '[') {
             sigmas_str.erase(0, 1);
         }
@@ -1507,7 +1573,7 @@ ArgOptions SDGenerationParams::get_options() {
         if (++index >= argc) {
             return -1;
         }
-        ref_image_paths.push_back(argv[index]);
+        ref_image_paths.push_back(argv_to_utf8(index, argv));
         return 1;
     };
 
@@ -1515,7 +1581,7 @@ ArgOptions SDGenerationParams::get_options() {
         if (++index >= argc) {
             return -1;
         }
-        ref_video_paths.push_back(argv[index]);
+        ref_video_paths.push_back(argv_to_utf8(index, argv));
         return 1;
     };
 
@@ -1523,7 +1589,7 @@ ArgOptions SDGenerationParams::get_options() {
         if (++index >= argc) {
             return -1;
         }
-        ref_video_audio_paths.push_back(argv[index]);
+        ref_video_audio_paths.push_back(argv_to_utf8(index, argv));
         return 1;
     };
 
@@ -1531,7 +1597,7 @@ ArgOptions SDGenerationParams::get_options() {
         if (++index >= argc) {
             return -1;
         }
-        ref_audio_paths.push_back(argv[index]);
+        ref_audio_paths.push_back(argv_to_utf8(index, argv));
         return 1;
     };
 
@@ -1539,7 +1605,7 @@ ArgOptions SDGenerationParams::get_options() {
         if (++index >= argc) {
             return -1;
         }
-        ref_audio_paths.push_back(argv[index]);
+        ref_audio_paths.push_back(argv_to_utf8(index, argv));
         return 1;
     };
 
@@ -1596,12 +1662,12 @@ ArgOptions SDGenerationParams::get_options() {
         size_t x_pos              = tile_size_str.find('x');
         try {
             if (x_pos != std::string::npos) {
-                std::string tile_x_str        = tile_size_str.substr(0, x_pos);
-                std::string tile_y_str        = tile_size_str.substr(x_pos + 1);
-                vae_tiling_params.tile_size_x = std::stoi(tile_x_str);
-                vae_tiling_params.tile_size_y = std::stoi(tile_y_str);
+                std::string tile_w_str        = tile_size_str.substr(0, x_pos);
+                std::string tile_h_str        = tile_size_str.substr(x_pos + 1);
+                vae_tiling_params.tile_size_w = std::stoi(tile_w_str);
+                vae_tiling_params.tile_size_h = std::stoi(tile_h_str);
             } else {
-                vae_tiling_params.tile_size_x = vae_tiling_params.tile_size_y = std::stoi(tile_size_str);
+                vae_tiling_params.tile_size_w = vae_tiling_params.tile_size_h = std::stoi(tile_size_str);
             }
         } catch (const std::invalid_argument&) {
             return -1;
@@ -1619,12 +1685,12 @@ ArgOptions SDGenerationParams::get_options() {
         size_t x_pos             = rel_size_str.find('x');
         try {
             if (x_pos != std::string::npos) {
-                std::string rel_x_str        = rel_size_str.substr(0, x_pos);
-                std::string rel_y_str        = rel_size_str.substr(x_pos + 1);
-                vae_tiling_params.rel_size_x = std::stof(rel_x_str);
-                vae_tiling_params.rel_size_y = std::stof(rel_y_str);
+                std::string rel_w_str        = rel_size_str.substr(0, x_pos);
+                std::string rel_h_str        = rel_size_str.substr(x_pos + 1);
+                vae_tiling_params.rel_size_w = std::stof(rel_w_str);
+                vae_tiling_params.rel_size_h = std::stof(rel_h_str);
             } else {
-                vae_tiling_params.rel_size_x = vae_tiling_params.rel_size_y = std::stof(rel_size_str);
+                vae_tiling_params.rel_size_w = vae_tiling_params.rel_size_h = std::stof(rel_size_str);
             }
         } catch (const std::invalid_argument&) {
             return -1;
@@ -1638,15 +1704,15 @@ ArgOptions SDGenerationParams::get_options() {
         if (++index >= argc) {
             return -1;
         }
-        const char* arg = argv[index];
-        std::ifstream f(arg, std::ios::binary);
+        const std::string arg = argv_to_utf8(index, argv);
+        std::ifstream f(fs::u8path(arg), std::ios::binary);
         try {
             prompt = std::string(std::istreambuf_iterator<char>{f}, {});
         } catch (const std::ios_base::failure&) {
             f.setstate(std::ios_base::failbit);
         }
         if (f.fail()) {
-            LOG_ERROR("error: failed to read prompt file '%s'\n", arg);
+            LOG_ERROR("error: failed to read prompt file '%s'\n", arg.c_str());
             return -1;
         }
         return 1;
@@ -1656,15 +1722,15 @@ ArgOptions SDGenerationParams::get_options() {
         if (++index >= argc) {
             return -1;
         }
-        const char* arg = argv[index];
-        std::ifstream f(arg, std::ios::binary);
+        const std::string arg = argv_to_utf8(index, argv);
+        std::ifstream f(fs::u8path(arg), std::ios::binary);
         try {
             negative_prompt = std::string(std::istreambuf_iterator<char>{f}, {});
         } catch (const std::ios_base::failure&) {
             f.setstate(std::ios_base::failbit);
         }
         if (f.fail()) {
-            LOG_ERROR("error: failed to read negative prompt file '%s'\n", arg);
+            LOG_ERROR("error: failed to read negative prompt file '%s'\n", arg.c_str());
             return -1;
         }
         return 1;
@@ -1702,11 +1768,11 @@ ArgOptions SDGenerationParams::get_options() {
          on_scheduler_arg},
         {"",
          "--sigmas",
-         "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\").",
+         "custom sigma values for the sampler, comma-separated (e.g., \"14.61,7.8,3.5,0.0\") or preset: qwen-image-2.1-turbo.",
          on_sigmas_arg},
         {"",
          "--hires-sigmas",
-         "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\").",
+         "custom sigma values for the highres fix second pass, comma-separated (e.g., \"0.85,0.725,0.421875,0.0\") or preset: qwen-image-2.1-turbo.",
          on_hires_sigmas_arg},
         {"",
          "--skip-layers",
@@ -1754,11 +1820,11 @@ ArgOptions SDGenerationParams::get_options() {
          on_scm_policy_arg},
         {"",
          "--vae-tile-size",
-         "tile size for vae tiling, format [X]x[Y] (default: 32x32)",
+         "tile size for vae encode and decode in image pixels, format [W]x[H] or [S] (default: 256x256; requires --vae-tiling)",
          on_tile_size_arg},
         {"",
          "--vae-relative-tile-size",
-         "relative tile size for vae tiling, format [X]x[Y], in fraction of image size if < 1, in number of tiles per dim if >=1 (overrides --vae-tile-size)",
+         "relative tile size for vae encode and decode, format [W]x[H] or [S]: <=1 is a dimension fraction, >1 a target tile count (overrides --vae-tile-size; requires --vae-tiling)",
          on_relative_tile_size_arg},
         {"",
          "--prompt-file",
@@ -1848,28 +1914,28 @@ bool decode_base64_image(const std::string& encoded_input,
         return false;
     }
 
-    int decoded_width  = 0;
-    int decoded_height = 0;
-    uint8_t* raw_data  = load_image_from_memory(reinterpret_cast<const char*>(image_bytes.data()),
-                                                static_cast<int>(image_bytes.size()),
-                                                decoded_width,
-                                                decoded_height,
-                                                expected_width,
-                                                expected_height,
-                                                target_channels);
+    int decoded_width    = 0;
+    int decoded_height   = 0;
+    int resolved_channel = target_channels;
+    uint8_t* raw_data    = load_image_from_memory(reinterpret_cast<const char*>(image_bytes.data()),
+                                                  static_cast<int>(image_bytes.size()),
+                                                  decoded_width,
+                                                  decoded_height,
+                                                  resolved_channel,
+                                                  expected_width,
+                                                  expected_height,
+                                                  target_channels);
     if (raw_data == nullptr) {
         return false;
     }
 
-    out_image.reset({(uint32_t)decoded_width, (uint32_t)decoded_height, (uint32_t)target_channels, raw_data});
+    out_image.reset({(uint32_t)decoded_width, (uint32_t)decoded_height, (uint32_t)resolved_channel, raw_data});
     return true;
 }
 
 static bool parse_image_json_field(const json& parent,
                                    const char* key,
                                    int channels,
-                                   int expected_width,
-                                   int expected_height,
                                    SDImageOwner& out_image) {
     if (!parent.contains(key)) {
         return true;
@@ -1881,14 +1947,12 @@ static bool parse_image_json_field(const json& parent,
     if (!parent.at(key).is_string()) {
         return false;
     }
-    return decode_base64_image(parent.at(key).get<std::string>(), channels, expected_width, expected_height, out_image);
+    return decode_base64_image(parent.at(key).get<std::string>(), channels, 0, 0, out_image);
 }
 
 static bool parse_image_array_json_field(const json& parent,
                                          const char* key,
                                          int channels,
-                                         int expected_width,
-                                         int expected_height,
                                          std::vector<SDImageOwner>& out_images) {
     if (!parent.contains(key)) {
         return true;
@@ -1907,7 +1971,7 @@ static bool parse_image_array_json_field(const json& parent,
             return false;
         }
         SDImageOwner image;
-        if (!decode_base64_image(item.get<std::string>(), channels, expected_width, expected_height, image)) {
+        if (!decode_base64_image(item.get<std::string>(), channels, 0, 0, image)) {
             return false;
         }
         out_images.push_back(std::move(image));
@@ -1987,25 +2051,48 @@ static bool resolve_model_file_from_dir(const std::string& model_name,
     if (model_name.empty() ||
         model_name.find('/') != std::string::npos ||
         model_name.find('\\') != std::string::npos ||
-        fs::path(model_name).has_root_path() ||
+        fs::u8path(model_name).has_root_path() ||
         ends_with_valid_ext()) {
         LOG_ERROR("%s must be a model name without path or extension: %s", label, model_name.c_str());
         return false;
     }
 
-    fs::path model_dir_path = model_dir;
+    fs::path model_dir_path = fs::u8path(model_dir);
     for (const auto& ext : valid_ext) {
-        fs::path try_path = model_dir_path / (model_name + ext);
+        fs::path try_path = model_dir_path / fs::u8path(model_name + ext);
         if (fs::exists(try_path) && fs::is_regular_file(try_path)) {
-            resolved_path = try_path.lexically_normal().string();
+            resolved_path = try_path.lexically_normal().u8string();
             return true;
         }
     }
 
-    LOG_ERROR("can not find %s %s in %s", label, model_name.c_str(), model_dir_path.lexically_normal().string().c_str());
+    LOG_ERROR("can not find %s %s in %s", label, model_name.c_str(), model_dir_path.lexically_normal().u8string().c_str());
     return false;
 }
 
+bool SDGenerationParams::parse_image_preprocess_json(const std::string& json_str) {
+    const auto value = json::parse(json_str, nullptr, false);
+    std::string rules;
+    if (value.is_string()) {
+        rules = value.get<std::string>();
+    } else if (value.is_array()) {
+        for (const auto& item : value) {
+            if (!item.is_string()) {
+                LOG_ERROR("image_preprocess must contain rule strings");
+                return false;
+            }
+            if (!rules.empty())
+                rules += ";";
+            rules += item.get<std::string>();
+        }
+    } else {
+        LOG_ERROR("image_preprocess must be a string or array of strings");
+        return false;
+    }
+    image_preprocess = std::move(rules);
+    return true;
+}
+
 bool SDGenerationParams::from_json_str(
     const std::string& json_str,
     const std::function<std::string(const std::string&)>& lora_path_resolver) {
@@ -2017,6 +2104,9 @@ bool SDGenerationParams::from_json_str(
         return false;
     }
 
+    if (j.contains("image_preprocess") && !parse_image_preprocess_json(j["image_preprocess"].dump()))
+        return false;
+
     auto load_if_exists = [&](const char* key, auto& out) {
         if (j.contains(key)) {
             using T = std::decay_t<decltype(out)>;
@@ -2054,6 +2144,7 @@ bool SDGenerationParams::from_json_str(
     load_if_exists("cache_mode", cache_mode);
     load_if_exists("cache_option", cache_option);
     load_if_exists("scm_mask", scm_mask);
+    load_if_exists("ref_image_args", ref_image_args);
 
     load_if_exists("clip_skip", clip_skip);
     load_if_exists("width", width);
@@ -2071,7 +2162,6 @@ bool SDGenerationParams::from_json_str(
     load_if_exists("moe_boundary", moe_boundary);
     load_if_exists("vace_strength", vace_strength);
 
-    load_if_exists("auto_resize_ref_image", auto_resize_ref_image);
     load_if_exists("increase_ref_index", increase_ref_index);
     load_if_exists("embed_image_metadata", embed_image_metadata);
 
@@ -2128,7 +2218,8 @@ bool SDGenerationParams::from_json_str(
         }
         if (target_custom_sigmas != nullptr &&
             sample_json.contains("custom_sigmas") &&
-            sample_json["custom_sigmas"].is_array()) {
+            sample_json["custom_sigmas"].is_array() &&
+            !sample_json["custom_sigmas"].empty()) { // empty [] = "not specified", keep server/CLI default
             *target_custom_sigmas = sample_json["custom_sigmas"].get<std::vector<float>>();
         }
         if (sample_json.contains("sample_method") && sample_json["sample_method"].is_string()) {
@@ -2191,20 +2282,20 @@ bool SDGenerationParams::from_json_str(
         if (tiling_json.contains("temporal_tiling") && tiling_json["temporal_tiling"].is_boolean()) {
             vae_tiling_params.temporal_tiling = tiling_json["temporal_tiling"];
         }
-        if (tiling_json.contains("tile_size_x") && tiling_json["tile_size_x"].is_number_integer()) {
-            vae_tiling_params.tile_size_x = tiling_json["tile_size_x"];
+        if (tiling_json.contains("tile_size_w") && tiling_json["tile_size_w"].is_number_integer()) {
+            vae_tiling_params.tile_size_w = tiling_json["tile_size_w"];
         }
-        if (tiling_json.contains("tile_size_y") && tiling_json["tile_size_y"].is_number_integer()) {
-            vae_tiling_params.tile_size_y = tiling_json["tile_size_y"];
+        if (tiling_json.contains("tile_size_h") && tiling_json["tile_size_h"].is_number_integer()) {
+            vae_tiling_params.tile_size_h = tiling_json["tile_size_h"];
         }
         if (tiling_json.contains("target_overlap") && tiling_json["target_overlap"].is_number()) {
             vae_tiling_params.target_overlap = tiling_json["target_overlap"];
         }
-        if (tiling_json.contains("rel_size_x") && tiling_json["rel_size_x"].is_number()) {
-            vae_tiling_params.rel_size_x = tiling_json["rel_size_x"];
+        if (tiling_json.contains("rel_size_w") && tiling_json["rel_size_w"].is_number()) {
+            vae_tiling_params.rel_size_w = tiling_json["rel_size_w"];
         }
-        if (tiling_json.contains("rel_size_y") && tiling_json["rel_size_y"].is_number()) {
-            vae_tiling_params.rel_size_y = tiling_json["rel_size_y"];
+        if (tiling_json.contains("rel_size_h") && tiling_json["rel_size_h"].is_number()) {
+            vae_tiling_params.rel_size_h = tiling_json["rel_size_h"];
         }
         if (tiling_json.contains("extra_tiling_args") && tiling_json["extra_tiling_args"].is_string()) {
             extra_tiling_args = tiling_json["extra_tiling_args"].get<std::string>();
@@ -2215,46 +2306,38 @@ bool SDGenerationParams::from_json_str(
         LOG_ERROR("invalid lora");
         return false;
     }
-    if (!parse_image_json_field(j, "init_image", 3, width, height, init_image)) {
-        LOG_ERROR("invalid init_image");
-        return false;
-    }
-    if (!parse_image_json_field(j, "end_image", 3, width, height, end_image)) {
-        LOG_ERROR("invalid end_image");
-        return false;
-    }
-    if (!parse_image_array_json_field(j, "ref_images", 3, width, height, ref_images)) {
-        LOG_ERROR("invalid ref_images");
-        return false;
-    }
-    if (!parse_image_array_json_field(j, "control_frames", 3, width, height, control_frames)) {
-        LOG_ERROR("invalid control_frames");
-        return false;
-    }
-    if (!parse_image_json_field(j, "mask_image", 1, width, height, mask_image)) {
-        LOG_ERROR("invalid mask_image");
-        return false;
-    }
-    if (!parse_image_json_field(j, "control_image", 3, width, height, control_image)) {
-        LOG_ERROR("invalid control_image");
+    auto load_image = [&](const char* key, int channels, SDImageOwner& image) {
+        if (!parse_image_json_field(j, key, channels, image)) {
+            LOG_ERROR("invalid %s", key);
+            return false;
+        }
+        return true;
+    };
+    if (!load_image("init_image", 0, init_image) ||
+        !load_image("end_image", 3, end_image) ||
+        !load_image("mask_image", 1, mask_image) ||
+        !load_image("control_image", 3, control_image) ||
+        !load_image("ip_adapter_image", 3, ip_adapter_image)) {
         return false;
     }
-    if (!parse_image_json_field(j, "ip_adapter_image", 3, width, height, ip_adapter_image)) {
-        LOG_ERROR("invalid ip_adapter_image");
+    if (!parse_image_array_json_field(j, "ref_images", 0, ref_images) ||
+        !parse_image_array_json_field(j, "control_frames", 3, control_frames)) {
+        LOG_ERROR("invalid input image array");
         return false;
     }
 
     return true;
 }
 
-void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) {
+bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) {
     if (lora_model_dir.empty()) {
-        return;
+        return true;
     }
     static const std::regex re(R"(<lora:([^:>]+):([^>]+)>)");
     static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt", ".ckpt"};
     std::smatch m;
 
+    bool missing    = false;
     std::string tmp = prompt;
 
     while (std::regex_search(tmp, m, re)) {
@@ -2278,10 +2361,18 @@ void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
         }
 
         fs::path final_path;
-        if (is_absolute_path(raw_path)) {
-            final_path = raw_path;
-        } else {
-            final_path = fs::path(lora_model_dir) / raw_path;
+        try {
+            if (is_absolute_path(raw_path)) {
+                final_path = fs::u8path(raw_path);
+            } else {
+                final_path = fs::u8path(lora_model_dir) / fs::u8path(raw_path);
+            }
+        } catch (const std::exception&) {  // not UTF-8, e.g. a prompt file in another encoding
+            LOG_WARN("invalid lora path %s", raw_path.c_str());
+            tmp     = m.suffix().str();
+            prompt  = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only);
+            missing = true;
+            continue;
         }
         if (!fs::exists(final_path)) {
             bool found = false;
@@ -2295,14 +2386,15 @@ void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
                 }
             }
             if (!found) {
-                LOG_WARN("can not found lora %s", final_path.lexically_normal().string().c_str());
-                tmp    = m.suffix().str();
-                prompt = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only);
+                LOG_WARN("can not found lora %s", final_path.lexically_normal().u8string().c_str());
+                tmp     = m.suffix().str();
+                prompt  = std::regex_replace(prompt, re, "", std::regex_constants::format_first_only);
+                missing = true;
                 continue;
             }
         }
 
-        const std::string key = final_path.lexically_normal().string();
+        const std::string key = final_path.lexically_normal().u8string();
 
         if (is_high_noise)
             high_noise_lora_map[key] += mul;
@@ -2313,6 +2405,8 @@ void SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
 
         tmp = m.suffix().str();
     }
+
+    return !missing;
 }
 
 bool SDGenerationParams::width_and_height_are_set() const {
@@ -2434,7 +2528,7 @@ bool SDGenerationParams::initialize_cache_params() {
     return true;
 }
 
-bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict) {
+bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict, bool validate_missing_loras) {
     vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
 
     if (high_noise_sample_params.sample_steps <= 0) {
@@ -2478,12 +2572,18 @@ bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::s
 
     prompt_with_lora = prompt;
     if (!lora_model_dir.empty()) {
-        extract_and_remove_lora(lora_model_dir);
+        if (!extract_and_remove_lora(lora_model_dir) && validate_missing_loras) {
+            return false;
+        }
     }
     return true;
 }
 
 bool SDGenerationParams::validate(SDMode mode) {
+    if (!image_preprocess.empty() && mode != IMG_GEN && mode != VID_GEN) {
+        LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode");
+        return false;
+    }
     if (batch_count <= 0) {
         LOG_ERROR("error: batch_count must be greater than 0");
         return false;
@@ -2602,7 +2702,7 @@ bool SDGenerationParams::resolve_and_validate(SDMode mode,
                                               const std::string& lora_model_dir,
                                               const std::string& hires_upscalers_dir,
                                               bool strict) {
-    if (!resolve(lora_model_dir, hires_upscalers_dir, strict)) {
+    if (!resolve(lora_model_dir, hires_upscalers_dir, strict, mode == CONVERT)) {
         return false;
     }
     if (!validate(mode)) {
@@ -2659,14 +2759,6 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() {
         pulid_id_weight,
     };
 
-    if (!auto_resize_ref_image) {
-        if (!ref_image_args.empty()) {
-            ref_image_args += ",";
-        }
-        ref_image_args += "resize_before_vae=0";
-        LOG_WARN("Notice: --disable-auto-resize-ref-image is deprecated. Use --ref-image-args \"resize_before_vae=off\" instead.");
-    }
-
     if (increase_ref_index) {
         if (!ref_image_args.empty()) {
             ref_image_args += ",";
@@ -2714,6 +2806,7 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() {
     params.hires.custom_sigmas_count = static_cast<int>(hires_custom_sigmas.size());
     params.circular_x                = circular || circular_x;
     params.circular_y                = circular || circular_y;
+    params.image_preprocess          = {image_preprocess.c_str()};
     return params;
 }
 
@@ -2816,6 +2909,7 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
     params.hires.custom_sigmas_count = static_cast<int>(hires_custom_sigmas.size());
     params.circular_x                = circular || circular_x;
     params.circular_y                = circular || circular_y;
+    params.image_preprocess          = {image_preprocess.c_str()};
     return params;
 }
 
@@ -2872,7 +2966,8 @@ std::string SDGenerationParams::to_string() const {
         << "  ref_video_audio_paths: " << vec_str_to_string(ref_video_audio_paths) << ",\n"
         << "  ref_audio_paths: " << vec_str_to_string(ref_audio_paths) << ",\n"
         << "  control_video_path: \"" << control_video_path << "\",\n"
-        << "  auto_resize_ref_image: " << (auto_resize_ref_image ? "true" : "false") << ",\n"
+        << "  image_preprocess: " << image_preprocess << ",\n"
+        << "  ref_image_args: " << ref_image_args << ",\n"
         << "  increase_ref_index: " << (increase_ref_index ? "true" : "false") << ",\n"
         << "  pm_id_images_dir: \"" << pm_id_images_dir << "\",\n"
         << "  pm_id_embed_path: \"" << pm_id_embed_path << "\",\n"
@@ -2911,11 +3006,11 @@ std::string SDGenerationParams::to_string() const {
         << "  vae_tiling_params: { "
         << vae_tiling_params.enabled << ", "
         << vae_tiling_params.temporal_tiling << ", "
-        << vae_tiling_params.tile_size_x << ", "
-        << vae_tiling_params.tile_size_y << ", "
+        << vae_tiling_params.tile_size_w << ", "
+        << vae_tiling_params.tile_size_h << ", "
         << vae_tiling_params.target_overlap << ", "
-        << vae_tiling_params.rel_size_x << ", "
-        << vae_tiling_params.rel_size_y << ", "
+        << vae_tiling_params.rel_size_w << ", "
+        << vae_tiling_params.rel_size_h << ", "
         << "\"" << extra_tiling_args << "\" },\n"
         << "}";
     return oss.str();
@@ -3023,12 +3118,13 @@ std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
     set_json_basename_if_not_empty(models, "control_net", ctx_params.control_net_path);
     root["models"] = std::move(models);
 
-    root["clip_skip"]             = gen_params.clip_skip;
-    root["strength"]              = gen_params.strength;
-    root["control_strength"]      = gen_params.control_strength;
-    root["ip_adapter_strength"]   = gen_params.ip_adapter_strength;
-    root["auto_resize_ref_image"] = gen_params.auto_resize_ref_image;
-    root["increase_ref_index"]    = gen_params.increase_ref_index;
+    root["clip_skip"]           = gen_params.clip_skip;
+    root["strength"]            = gen_params.strength;
+    root["control_strength"]    = gen_params.control_strength;
+    root["ip_adapter_strength"] = gen_params.ip_adapter_strength;
+    root["ref_image_args"]      = gen_params.ref_image_args;
+    root["image_preprocess"]    = gen_params.image_preprocess;
+    root["increase_ref_index"]  = gen_params.increase_ref_index;
     if (mode == VID_GEN) {
         root["video"] = {
             {"frame_count", gen_params.video_frames},
@@ -3116,11 +3212,11 @@ std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params,
         root["vae_tiling"] = {
             {"enabled", gen_params.vae_tiling_params.enabled},
             {"temporal_tiling", gen_params.vae_tiling_params.temporal_tiling},
-            {"tile_size_x", gen_params.vae_tiling_params.tile_size_x},
-            {"tile_size_y", gen_params.vae_tiling_params.tile_size_y},
+            {"tile_size_w", gen_params.vae_tiling_params.tile_size_w},
+            {"tile_size_h", gen_params.vae_tiling_params.tile_size_h},
             {"target_overlap", gen_params.vae_tiling_params.target_overlap},
-            {"rel_size_x", gen_params.vae_tiling_params.rel_size_x},
-            {"rel_size_y", gen_params.vae_tiling_params.rel_size_y},
+            {"rel_size_w", gen_params.vae_tiling_params.rel_size_w},
+            {"rel_size_h", gen_params.vae_tiling_params.rel_size_h},
             {"extra_tiling_args", gen_params.extra_tiling_args},
         };
     }
diff --git a/examples/common/common.h b/examples/common/common.h
index 0e5c50e3cb..91e75c6851 100644
--- a/examples/common/common.h
+++ b/examples/common/common.h
@@ -12,6 +12,10 @@
 #include "resource_owners.hpp"
 #include "stable-diffusion.h"
 
+// The command-line argument at `index` as UTF-8 (on Windows, read again from
+// the UTF-16 command line: argv itself is in the ANSI code page).
+std::string argv_to_utf8(int index, const char** argv);
+
 #define SAFE_STR(s) ((s) ? (s) : "")
 #define BOOL_STR(b) ((b) ? "true" : "false")
 
@@ -116,7 +120,8 @@ bool decode_base64_image(const std::string& encoded_input,
                          SDImageOwner& out_image);
 
 struct SDContextParams {
-    int n_threads = -1;
+    int n_threads               = -1;
+    int conditioning_cache_size = 4;
     std::string model_path;
     std::string clip_l_path;
     std::string clip_g_path;
@@ -156,6 +161,7 @@ struct SDContextParams {
     bool disable_prefetch          = false;
     bool disable_segmented_compute = false;
     bool eager_load                = false;
+    bool batched_cfg               = true;
     std::string backend;
     std::string params_backend;
     std::string split_mode;
@@ -164,12 +170,12 @@ struct SDContextParams {
     std::string rpc_servers;
     std::string effective_backend;
     std::string effective_params_backend;
-    bool enable_mmap           = false;
-    bool control_net_cpu       = false;
-    bool clip_on_cpu           = false;
-    bool vae_on_cpu            = false;
-    bool flash_attn            = false;
-    bool diffusion_flash_attn  = false;
+    bool enable_mmap            = false;
+    bool control_net_cpu        = false;
+    bool clip_on_cpu            = false;
+    bool vae_on_cpu             = false;
+    std::string flash_attn_mode = "auto";
+    std::string conv_direct_mode;  // Empty selects auto while honoring legacy convolution flags.
     bool sage_attn             = false;
     bool diffusion_conv_direct = false;
     bool vae_conv_direct       = false;
@@ -200,18 +206,17 @@ struct SDGenerationParams {
     std::string ad_prompt;
     std::string ad_negative_prompt;
     std::string extra_ad_args;
-    int clip_skip              = -1;  // <= 0 represents unspecified
-    int width                  = -1;
-    int height                 = -1;
-    int batch_count            = 1;
-    int qwen_image_layers      = 3;
-    int64_t seed               = 42;
-    float strength             = 0.75f;
-    float control_strength     = 0.9f;
-    float ip_adapter_strength  = 1.0f;
-    bool auto_resize_ref_image = true;
-    bool increase_ref_index    = false;
-    bool embed_image_metadata  = true;
+    int clip_skip             = -1;  // <= 0 represents unspecified
+    int width                 = -1;
+    int height                = -1;
+    int batch_count           = 1;
+    int qwen_image_layers     = 3;
+    int64_t seed              = 42;
+    float strength            = 0.75f;
+    float control_strength    = 0.9f;
+    float ip_adapter_strength = 1.0f;
+    bool increase_ref_index   = false;
+    bool embed_image_metadata = true;
 
     std::string init_image_path;
     std::string end_image_path;
@@ -247,6 +252,7 @@ struct SDGenerationParams {
     std::string extra_tiling_args;
 
     std::string ref_image_args;
+    std::string image_preprocess;
 
     std::string pm_id_images_dir;
     std::string pm_id_embed_path;
@@ -310,13 +316,14 @@ struct SDGenerationParams {
     ArgOptions get_options();
     bool from_json_str(const std::string& json_str,
                        const std::function<std::string(const std::string&)>& lora_path_resolver = {});
+    bool parse_image_preprocess_json(const std::string& json_str);
     bool initialize_cache_params();
-    void extract_and_remove_lora(const std::string& lora_model_dir);
+    bool extract_and_remove_lora(const std::string& lora_model_dir);
     bool width_and_height_are_set() const;
     void set_width_and_height_if_unset(int w, int h);
     int get_resolved_width() const;
     int get_resolved_height() const;
-    bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false);
+    bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false, bool validate_missing_loras = false);
     bool validate(SDMode mode);
     bool resolve_and_validate(SDMode mode,
                               const std::string& lora_model_dir,
diff --git a/examples/common/log.cpp b/examples/common/log.cpp
index c272c2376d..12d0683274 100644
--- a/examples/common/log.cpp
+++ b/examples/common/log.cpp
@@ -91,34 +91,34 @@ void log_print(enum sd_log_level_t level, const char* log, sd_log_level_t min_le
     switch (level) {
         case SD_LOG_DEBUG:
             tag_color = 37;
-            level_str = "DEBUG";
+            level_str = "D";
             break;
         case SD_LOG_VERBOSE:
             tag_color = 37;
-            level_str = "VERBOSE";
+            level_str = "V";
             break;
         case SD_LOG_INFO:
             tag_color = 34;
-            level_str = "INFO";
+            level_str = "I";
             break;
         case SD_LOG_WARN:
             tag_color = 35;
-            level_str = "WARN";
+            level_str = "W";
             break;
         case SD_LOG_ERROR:
             tag_color = 31;
-            level_str = "ERROR";
+            level_str = "E";
             break;
         default:
             tag_color = 33;
-            level_str = "?????";
+            level_str = "?";
             break;
     }
 
     if (color) {
-        fprintf(out_stream, "\033[%d;1m[%-7s]\033[0m ", tag_color, level_str);
+        fprintf(out_stream, "\033[%d;1m[%s]\033[0m ", tag_color, level_str);
     } else {
-        fprintf(out_stream, "[%-7s] ", level_str);
+        fprintf(out_stream, "[%s] ", level_str);
     }
     fflush(out_stream);
     print_utf8(out_stream, log);
@@ -131,18 +131,15 @@ void example_log_printf(sd_log_level_t level, const char* file, int line, const
     va_list args;
     va_start(args, format);
 
-    static char log_buffer[LOG_BUFFER_SIZE + 1];
-    int written = snprintf(log_buffer, LOG_BUFFER_SIZE, "%s:%-4d - ", sd_basename(file).c_str(), line);
+    char log_buffer[LOG_BUFFER_SIZE + 1] = {};
+    vsnprintf(log_buffer, sizeof(log_buffer), format, args);
+    va_end(args);
 
-    if (written >= 0 && written < static_cast<int>(LOG_BUFFER_SIZE)) {
-        vsnprintf(log_buffer + written, LOG_BUFFER_SIZE - written, format, args);
-    }
-    size_t len = strlen(log_buffer);
-    if (len == 0 || log_buffer[len - 1] != '\n') {
-        strncat(log_buffer, "\n", LOG_BUFFER_SIZE - len);
+    std::string message = log_buffer;
+    while (!message.empty() && (message.back() == '\n' || message.back() == '\r')) {
+        message.pop_back();
     }
+    message += " --- " + sd_basename(file) + ":" + std::to_string(line) + '\n';
 
-    log_print(level, log_buffer, log_level, log_color);
-
-    va_end(args);
+    log_print(level, message.c_str(), log_level, log_color);
 }
diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp
index 345b6f1523..baac6d2009 100644
--- a/examples/common/media_io.cpp
+++ b/examples/common/media_io.cpp
@@ -15,10 +15,12 @@
 
 #define STB_IMAGE_IMPLEMENTATION
 #define STB_IMAGE_STATIC
+#define STBI_WINDOWS_UTF8
 #include "stb_image.h"
 
 #define STB_IMAGE_WRITE_IMPLEMENTATION
 #define STB_IMAGE_WRITE_STATIC
+#define STBIW_WINDOWS_UTF8
 #include "stb_image_write.h"
 
 #define STB_IMAGE_RESIZE_IMPLEMENTATION
@@ -147,7 +149,7 @@ class MemoryMkvWriter : public mkvmuxer::IMkvWriter {
 #endif
 
 bool read_binary_file_bytes(const char* path, std::vector<uint8_t>& data) {
-    std::ifstream fin(fs::path(path), std::ios::binary);
+    std::ifstream fin(fs::u8path(path), std::ios::binary);
     if (!fin) {
         return false;
     }
@@ -170,7 +172,7 @@ bool read_binary_file_bytes(const char* path, std::vector<uint8_t>& data) {
 }
 
 bool write_binary_file_bytes(const std::string& path, const std::vector<uint8_t>& data) {
-    std::ofstream fout(fs::path(path), std::ios::binary);
+    std::ofstream fout(fs::u8path(path), std::ios::binary);
     if (!fout) {
         return false;
     }
@@ -261,6 +263,10 @@ uint8_t* decode_webp_image_to_buffer(const uint8_t* data,
     height               = features.height;
     source_channel_count = features.has_alpha ? 4 : 3;
 
+    if (expected_channel == 0) {
+        expected_channel = source_channel_count;
+    }
+
     const size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
 
     if (expected_channel == 1) {
@@ -481,7 +487,8 @@ uint8_t* load_image_common(bool from_memory,
                            int& height,
                            int expected_width,
                            int expected_height,
-                           int expected_channel) {
+                           int expected_channel,
+                           int& out_channel) {
     const char* image_path;
     FreeUniquePtr<uint8_t> image_buffer;
     int source_channel_count = 0;
@@ -538,6 +545,32 @@ uint8_t* load_image_common(bool from_memory,
         LOG_ERROR("load image from '%s' failed", image_path);
         return nullptr;
     }
+    if (expected_channel == 0) {
+        expected_channel = source_channel_count == 2 ? 4 : (source_channel_count == 1 ? 3 : source_channel_count);
+        if (expected_channel != source_channel_count) {
+            FreeUniquePtr<uint8_t> promoted((uint8_t*)malloc((size_t)width * height * expected_channel));
+            if (promoted == nullptr) {
+                LOG_ERROR("error: allocate memory for channel promotion, image_path = %s", image_path);
+                return nullptr;
+            }
+            const size_t pixel_count = (size_t)width * (size_t)height;
+            for (size_t i = 0; i < pixel_count; ++i) {
+                if (source_channel_count == 1) {
+                    promoted.get()[i * 3 + 0] = image_buffer.get()[i];
+                    promoted.get()[i * 3 + 1] = image_buffer.get()[i];
+                    promoted.get()[i * 3 + 2] = image_buffer.get()[i];
+                } else {
+                    promoted.get()[i * 4 + 0] = image_buffer.get()[i * 2];
+                    promoted.get()[i * 4 + 1] = image_buffer.get()[i * 2];
+                    promoted.get()[i * 4 + 2] = image_buffer.get()[i * 2];
+                    promoted.get()[i * 4 + 3] = image_buffer.get()[i * 2 + 1];
+                }
+            }
+            image_buffer         = std::move(promoted);
+            source_channel_count = expected_channel;
+        }
+    }
+    // stb reports the source channel count even when it converts the output.
     if (source_channel_count < expected_channel) {
         fprintf(stderr,
                 "the number of channels for the input image must be >= %d,"
@@ -597,7 +630,7 @@ uint8_t* load_image_common(bool from_memory,
         }
         stbir_resize(image_buffer.get(), width, height, 0,
                      resized_image_buffer.get(), expected_width, expected_height, 0, STBIR_TYPE_UINT8,
-                     expected_channel, STBIR_ALPHA_CHANNEL_NONE, 0,
+                     expected_channel, expected_channel == 4 ? 3 : STBIR_ALPHA_CHANNEL_NONE, 0,
                      STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP,
                      STBIR_FILTER_BOX, STBIR_FILTER_BOX,
                      STBIR_COLORSPACE_SRGB, nullptr);
@@ -605,6 +638,7 @@ uint8_t* load_image_common(bool from_memory,
         height       = expected_height;
         image_buffer = std::move(resized_image_buffer);
     }
+    out_channel = expected_channel;
     return image_buffer.release();
 }
 
@@ -682,7 +716,7 @@ static std::pair<uint64_t, uint64_t> audio_sample_range_for_video_frame(const sd
 }
 
 EncodedImageFormat encoded_image_format_from_path(const std::string& path) {
-    std::string ext = fs::path(path).extension().string();
+    std::string ext = fs::u8path(path).extension().u8string();
     std::transform(ext.begin(), ext.end(), ext.begin(), ::tolower);
 
     if (ext == ".jpg" || ext == ".jpeg" || ext == ".jpe") {
@@ -777,10 +811,11 @@ bool write_image_to_file(const std::string& path,
 uint8_t* load_image_from_file(const char* image_path,
                               int& width,
                               int& height,
+                              int& out_channel,
                               int expected_width,
                               int expected_height,
                               int expected_channel) {
-    return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel);
+    return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, out_channel);
 }
 
 bool load_sd_image_from_file(sd_image_t* image,
@@ -790,13 +825,14 @@ bool load_sd_image_from_file(sd_image_t* image,
                              int expected_channel) {
     int width;
     int height;
-    image->data = load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel);
+    int resolved_channel = expected_channel;
+    image->data          = load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, resolved_channel);
     if (image->data == nullptr) {
         return false;
     }
     image->width   = width;
     image->height  = height;
-    image->channel = expected_channel;
+    image->channel = resolved_channel;
     return true;
 }
 
@@ -804,10 +840,11 @@ uint8_t* load_image_from_memory(const char* image_bytes,
                                 int len,
                                 int& width,
                                 int& height,
+                                int& out_channel,
                                 int expected_width,
                                 int expected_height,
                                 int expected_channel) {
-    return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel);
+    return load_image_common(true, image_bytes, len, width, height, expected_width, expected_height, expected_channel, out_channel);
 }
 
 static void append_avi_metadata(std::vector<uint8_t>& data, const std::string& parameters) {
@@ -1378,7 +1415,7 @@ bool write_wav_to_file(const std::string& path,
         return false;
     }
 
-    std::ofstream file(path, std::ios::binary);
+    std::ofstream file(fs::u8path(path), std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -1440,7 +1477,7 @@ bool load_wav_from_file(const std::string& path,
     sample_rate = 0;
     channels    = 0;
 
-    std::ifstream file(path, std::ios::binary);
+    std::ifstream file(fs::u8path(path), std::ios::binary);
     uint8_t riff_header[12];
     if (!file.read(reinterpret_cast<char*>(riff_header), sizeof(riff_header)) ||
         std::memcmp(riff_header, "RIFF", 4) != 0 ||
diff --git a/examples/common/media_io.h b/examples/common/media_io.h
index 6adbe98758..b2b56eccde 100644
--- a/examples/common/media_io.h
+++ b/examples/common/media_io.h
@@ -32,9 +32,12 @@ bool write_image_to_file(const std::string& path,
                          const std::string& parameters = "",
                          int quality                   = 90);
 
+// expected_channel == 0 preserves native channels (grayscale -> RGB, gray+alpha -> RGBA).
+// out_channel receives the output channel count.
 uint8_t* load_image_from_file(const char* image_path,
                               int& width,
                               int& height,
+                              int& out_channel,
                               int expected_width   = 0,
                               int expected_height  = 0,
                               int expected_channel = 3);
@@ -49,6 +52,7 @@ uint8_t* load_image_from_memory(const char* image_bytes,
                                 int len,
                                 int& width,
                                 int& height,
+                                int& out_channel,
                                 int expected_width   = 0,
                                 int expected_height  = 0,
                                 int expected_channel = 3);
diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt
index 5c462258bc..58260be0d4 100644
--- a/examples/server/CMakeLists.txt
+++ b/examples/server/CMakeLists.txt
@@ -21,21 +21,21 @@ if(SD_SERVER_BUILD_FRONTEND AND EXISTS "${FRONTEND_DIR}")
         set(HAVE_FRONTEND_BUILD ON)
 
         add_custom_target(${TARGET}_frontend_install
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" install
+            COMMAND "${PNPM_EXECUTABLE}" --ignore-workspace -C "${FRONTEND_DIR}" install
             WORKING_DIRECTORY "${FRONTEND_DIR}"
             COMMENT "Installing frontend dependencies"
             VERBATIM
         )
 
         add_custom_target(${TARGET}_frontend_build
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" run build
+            COMMAND "${PNPM_EXECUTABLE}" --ignore-workspace -C "${FRONTEND_DIR}" run build
             WORKING_DIRECTORY "${FRONTEND_DIR}"
             COMMENT "Building frontend"
             VERBATIM
         )
 
         add_custom_target(${TARGET}_frontend_header
-            COMMAND "${PNPM_EXECUTABLE}" -C "${FRONTEND_DIR}" run build:header
+            COMMAND "${PNPM_EXECUTABLE}" --ignore-workspace -C "${FRONTEND_DIR}" run build:header
             WORKING_DIRECTORY "${FRONTEND_DIR}"
             COMMENT "Generating gen_index_html.h"
             VERBATIM
@@ -68,6 +68,8 @@ add_executable(${TARGET}
     ../common/media_io.cpp
     main.cpp
     runtime.cpp
+    model_catalog.cpp
+    routes_models.cpp
     async_jobs.cpp
     routes_index.cpp
     routes_openai.cpp
@@ -88,6 +90,17 @@ else()
     message(STATUS "HAVE_INDEX_HTML disabled")
 endif()
 
+file(GLOB BUILTIN_RECIPE_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/recipes/*.json")
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${BUILTIN_RECIPE_FILES})
+# Keep recipes in separate literals to stay within MSVC's string size limit.
+set(SD_BUILTIN_RECIPES "")
+foreach(RECIPE_FILE IN LISTS BUILTIN_RECIPE_FILES)
+    file(READ "${RECIPE_FILE}" RECIPE_JSON)
+    string(APPEND SD_BUILTIN_RECIPES "R\"recipes(${RECIPE_JSON})recipes\",\n")
+endforeach()
+configure_file(recipes_builtin.h.in "${CMAKE_CURRENT_BINARY_DIR}/recipes_builtin.h" @ONLY)
+target_include_directories(${TARGET} PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")
+
 install(TARGETS ${TARGET} RUNTIME)
 target_link_libraries(${TARGET} PRIVATE stable-diffusion ${CMAKE_THREAD_LIBS_INIT})
 if(SD_WEBP)
diff --git a/examples/server/README.md b/examples/server/README.md
index c59fc8aeb9..1bb49a59f5 100644
--- a/examples/server/README.md
+++ b/examples/server/README.md
@@ -1,9 +1,254 @@
+# Model selection
+
+Start without model arguments to select models in the web UI:
+
+```sh
+sd-server --model-dir ./models
+```
+
+From a source checkout, you can also use `sh ./start-server.sh` on Linux/macOS
+or `.\start-server.bat` on Windows. The launchers look for the server relative
+to the script: beside the script, then in `bin/Release/`, `bin/`,
+`build/bin/Release/`, and `build/bin/`, in that order.
+For a different build directory, set `SD_SERVER_PATH` to the executable path:
+
+```sh
+SD_SERVER_PATH="./build_vulkan/bin/sd-server" sh ./start-server.sh --listen-port 8080
+```
+
+```powershell
+$env:SD_SERVER_PATH = ".\build_vulkan\bin\Release\sd-server.exe"
+.\start-server.bat --listen-port 8080
+```
+
+Binary release archives include both launchers. Keep the launchers under version
+control and put personal settings in a separate file beside them:
+
+- Windows batch: `start-server.local.bat`
+- Linux/macOS: `start-server.local.sh`
+
+These local files are ignored by Git and excluded from release archives. They
+are loaded automatically after the launcher's defaults. They can override
+`SD_SERVER_PATH`, `MODEL_DIR`, and `COMMANDLINE_ARGS`; there is no need to edit
+the tracked launchers. For example:
+
+```bat
+@echo off
+set "MODEL_DIR=D:\stable-diffusion-webui\models"
+set "COMMANDLINE_ARGS=--listen-port 8080 --offload-to-cpu"
+```
+
+```sh
+MODEL_DIR="/data/stable-diffusion-webui/models"
+COMMANDLINE_ARGS="--listen-port 8080 --offload-to-cpu"
+```
+
+Each launcher prints the resolved command before execution and forwards the
+server's exit code.
+
+`SD_SERVER_PATH` inherits the environment value by default; leave it empty for
+automatic discovery, or set it in the local file to select another executable.
+Settings are applied in this order: launcher defaults, local configuration,
+then invocation arguments. Repeated `--model-dir` arguments add roots, so they
+do not replace the `MODEL_DIR` root. Local configuration files are shell scripts
+and use the corresponding shell's syntax.
+
+The launchers preserve the current working directory, so the default
+`MODEL_DIR="./models"` is relative to where you start the launcher. Arguments
+passed when invoking the launcher follow these defaults. Repeated `--model-dir`
+arguments add search roots rather than replacing `MODEL_DIR`; set `MODEL_DIR` in
+the local configuration to change the first root. Other single-value options can override the defaults.
+In the shell script, `COMMANDLINE_ARGS` is a
+whitespace-separated list; pass values containing spaces as quoted arguments
+when invoking the script. `MODEL_DIR` itself supports spaces in both scripts.
+A leading `--version` bypasses the configured defaults. An explicit
+`SD_SERVER_PATH` is used without falling back to another build if it is invalid.
+
+Choose a recipe, select the required component files, and click **Load models**.
+Recipe and component selectors support searching by name or relative file ID.
+Use the arrow keys and Enter to select, or Escape to cancel a search. With
+**Apply recipe defaults** enabled, selecting a recipe immediately applies its
+declared generation defaults while preserving prompts, input images and fields
+not specified by the recipe. Disable it to keep your current parameters when
+switching recipes. This preference defaults to enabled and is saved locally.
+Model loading and server-info refreshes do not reapply defaults or reset the form.
+Recipes declaring `video_frames` also switch the form to Video Generation;
+other recipes switch to Image Generation. Generation is enabled after the
+selected recipe is loaded. Changing a selection does not reload the model
+until you click Load.
+The server can also start with the existing model arguments shown below;
+existing API clients then work without selecting a recipe. The UI automatically
+selects **Command-line model (all features)** and uses the startup generation
+defaults. This runtime-only recipe exposes all controls for the supported
+image/video mode. Selecting another recipe previews a model switch; selecting
+the command-line entry again restores its generation defaults. The entry is
+removed when the command-line context is unloaded or replaced.
+
+Built-in recipes cover SD 1/2/3/3.5, SDXL and distilled checkpoints;
+FLUX 1/2/3, Chroma, Qwen Image and editing variants; Z-Image, Anima, ERNIE,
+LENS, PiD and other supported image families; AnimateDiff, Wan, LTX,
+HunyuanVideo, LingBot Video and MiniMax-H3. Model-specific text encoder
+labels identify the expected encoder family and size. Loading a video model enables **Video Generation**; I2V also requires a
+start image. The Models panel can be collapsed and remembers its open state.
+Its header shows elapsed loading time from the status API and keeps the final
+time after success or failure, including after a page refresh.
+Built-ins are embedded in the
+binary from [recipes/](recipes/), with one JSON file per recipe. Adding or
+editing a built-in requires rebuilding the server. User recipes are read from
+`<model-dir>/recipes/*.json`. A valid user recipe with the same ID replaces
+the entire built-in recipe. Across roots, the first valid user recipe wins;
+overrides are logged. Duplicate user IDs within one root are rejected; invalid
+recipes are reported without removing a valid recipe from another root or the
+built-in version. Slots need only `id` and
+`required`; the server supplies labels and search directories. Use `label`
+only when a recipe needs a more specific display name.
+
+Recipes use `features` to enable specialized controls and `disabled_features`
+to turn off shared defaults. Disabled features take precedence. The UI hides
+unsupported fields, empty sections and irrelevant sampler/cache options, and
+omits hidden values from generation requests. See the
+[feature declarations](api.md#recipe-feature-declarations) for the defaults
+and complete list of supported names.
+
+Built-in additions and changes must follow the [recipe policy](recipes/README.md).
+Split recipes by task weights or added capabilities, not by style or acceleration
+LoRAs. Quantizations, file formats and compatible layer-count variants share a
+recipe. Native encoders, projectors and connectors stay in the model's recipe.
+Community editing recipes require selecting the matching LoRA separately; a
+recipe does not enable it automatically.
+
+See [generation defaults and sources](recipes/defaults.md) for the selected
+publisher recommendations and reference-code defaults, with step defaults capped
+at 20. Recipes link to the
+original publisher's Hugging Face repository where one is available. DiT
+recipes default to Euler, except for models requiring a specialized sampler.
+
+Compatible recipes offer a TAE slot alongside VAE. When the checkpoint does not
+bundle a VAE, select at least one of VAE or TAE. The Models panel shows a reminder
+below the model selectors until this requirement is met. A selected TAE is used for
+encoding and decoding, including when a full VAE is also selected; use complete,
+model-compatible TAE weights rather than decoder-only preview weights. Qwen
+Image 2.1 uses the same recipe for its full VAE and TAEQI2.1 alternatives.
+
+The following variants have dedicated recipes:
+
+| Model | Recipe IDs / notes |
+| --- | --- |
+| Z-Image L2P | `z-image-l2p`; no VAE |
+| LLaDA-Image | `llada-image` (base) and `llada-image-turbo`; use matching encoders and connectors |
+| PixArt Alpha | `pixart-alpha-512`, `pixart-alpha-1024`; Alpha VAE scale and positional settings |
+| PiD / PiD 1.5 | `pid` / `pid-15` for Flux or Z-Image VAE; append `-sd3`, `-flux2`, or `-qwen-image` for other latent formats |
+| SDXL adapters | `sdxl-ip-adapter` supports classic and Plus ViT-H adapters; `sdxl-controlnet` requires compatible SDXL weights |
+| FLUX.1 Fill / Depth | `flux1-fill` uses Init Image and Mask Image; `flux1-depth` uses a precomputed depth map in Control Image |
+| SD2 / SDXL inpainting | `sd2-inpainting`, `sdxl-inpainting` |
+| SD-Turbo | `sd-turbo`; single-file SD2.1-derived checkpoint |
+| Community editing LoRAs | `anima-edit`, `krea2-raw-edit`, `krea2-turbo-edit`; apply a matching LoRA after loading |
+| HunyuanVideo 1.5 | `hunyuan-video-15-t2v`, `hunyuan-video-15-i2v`; choose the task's checkpoint |
+| Wan 2.1 / compatible single Wan 2.2 experts | `wan21-t2v`, `wan21-i2v`; the I2V recipe also accepts compatible SkyReels V2 weights |
+| Qwen Image 2.1 Turbo | `qwen-image-21-turbo`; standalone Turbo weights with the fixed 8-step schedule, Euler and CFG 1 |
+| MOSS Image | `moss-image-base`, `moss-image-sft`; choose the corresponding checkpoint |
+
+PiD always requires a reference image and uses the first one. Its recipes expose
+reference inputs rather than ordinary image-to-image inputs. IP-Adapter recipes
+expose a separate IP-Adapter image and strength. FLUX Canny/Depth consume Control
+Image directly without a separate ControlNet; the UI does not calculate edge
+or depth maps. Their strength is controlled by guidance, not ControlNet strength.
+Krea2 edit recipes use the default `krea2_ostris_edit` reference preset, so the
+editing LoRA must match that convention. Ordinary Anima and Krea2 recipes do not
+expose reference-image editing inputs.
+
+Fill/Depth guidance defaults follow the [Diffusers FLUX documentation](https://huggingface.co/docs/diffusers/api/pipelines/flux).
+SD-Turbo uses the [model card's one-step configuration](https://huggingface.co/stabilityai/sd-turbo),
+with CFG disabled using scale 1 in sd.cpp.
+
+SVD is not included: legacy model recognition and UNet branches remain, but the
+current runner/conditioning setup does not provide a dedicated SVD image-conditioning
+path. Adding a recipe alone would not establish a working SVD pipeline.
+
+Reference images, masks, audio, first/last video frames, and generation options
+such as upscaling remain generation inputs. Hardware and memory options remain
+startup arguments. Tiny SD/Tiny SD2 and Diffusers LCM directories must first be
+converted into single-file checkpoints for the file catalog. LCM checkpoints
+and LoRAs use the matching SD family recipe; select LCM sampling and adjust
+steps/CFG for the chosen weights.
+Metadata inspection, conversion, standalone upscaling and ADetailer are CLI
+workflows rather than model-loading recipes. ESRGAN and LTX latent upscalers
+are selected through generation upscaling settings with `--hires-upscalers-dir`;
+textual inversion uses the startup `--embd-dir` option.
+
+Repeat `--model-dir` to share multiple model directories, including an existing
+SD WebUI model directory:
+
+```sh
+sd-server --model-dir ./models --model-dir /data/stable-diffusion-webui/models
+```
+
+With no `--model-dir`, the server uses `./models`. Explicit roots replace that
+default and are scanned in argument order. Duplicate canonical roots and file
+paths are scanned once. Every root can contain a `recipes/` directory.
+Directory names are matched case-insensitively on all platforms:
+
+| Category | Directory names |
+| --- | --- |
+| Checkpoints | `checkpoints`, `Stable-diffusion` |
+| VAE | `vae`, `VAE` |
+| LoRA | `loras`, `Lora` |
+| ControlNet | `controlnet`, `ControlNet` |
+
+Other categories keep their existing directory names. Classification uses only
+the containing directory: `*.vae.safetensors`, `*.vae.ckpt`, and similar names
+inside `checkpoints/` or `Stable-diffusion/` remain checkpoints.
+
+Place model files in category directories, for example:
+
+```text
+models/
+  checkpoints/
+  diffusion_models/
+  text_encoders/
+  vae/
+  clip_vision/
+  loras/
+  recipes/
+    my-flux.json
+```
+
+Subdirectories within a model category are supported. Use **Refresh files &
+recipes** after adding files or changing recipes. Scanning lists file names
+without loading model weights. Category membership does not guarantee
+compatibility; the model loader performs the final compatibility check.
+Symlinks resolving outside the model directory are rejected. Dropdown labels
+omit the category prefix while retaining subdirectory names. Identical display
+paths are displayed as full file paths, with source directories before the
+filename instead of a hash suffix. The same applies to duplicate LoRA paths. File IDs include a
+root qualifier for additional roots; clients should use the returned IDs.
+Empty optional categories do not display missing-file hints.
+
+Place Tiny AutoEncoder (TAE) files in `vae/` and select them in the **TAE**
+slot. The legacy `vae_approx/` directory is also scanned for TAE selections.
+
+LoRAs are discovered automatically in both `loras/` and `Lora/` under every
+model root. Use `--lora-model-dir` to replace automatic discovery with one
+explicit LoRA directory. Duplicate names remain separate selectable entries. The LoRA selectors support
+searching by name or path with the same keyboard controls as model selectors.
+
+The server keeps one model context loaded. Loading or unloading is rejected
+while generation requests or queued jobs are pending. A switch validates the
+selection before unloading the old context, then loads the replacement.
+If loading fails, the service stays online with no active model and reports
+the error. Backend and memory options supplied at startup apply to subsequent
+loads. Recipe defaults apply to new loads; explicitly supplied generation
+request parameters take precedence.
+
+See [the API reference](api.md#model-management) for recipe fields, supported
+slots, management endpoints, and compatibility behavior.
+
 # Example
 
 The following example starts `sd-server` with a standalone diffusion model, VAE, and LLM text encoder:
 
 ```
-.\bin\Release\sd-server.exe --diffusion-model  ..\models\diffusion_models\z_image_turbo_bf16.safetensors --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\qwen_3_4b.safetensors --diffusion-fa --offload-to-cpu -v --cfg-scale 1.0
+.\bin\Release\sd-server.exe --diffusion-model  ..\models\diffusion_models\z_image_turbo_bf16.safetensors --vae ..\models\vae\ae.sft  --llm ..\models\text_encoders\qwen_3_4b.safetensors --offload-to-cpu -v --cfg-scale 1.0
 ```
 
 What this example does:
@@ -11,7 +256,6 @@ What this example does:
 * `--diffusion-model` selects the standalone diffusion model
 * `--vae` selects the VAE decoder
 * `--llm` selects the text encoder / language model used by this pipeline
-* `--diffusion-fa` enables flash attention in the diffusion model
 * `--offload-to-cpu` reduces VRAM pressure by keeping weights in RAM when possible
 * `-v` enables verbose logging (equivalent to `--log-level verbose`)
 * `--cfg-scale 1.0` sets the default CFG scale for generation
diff --git a/examples/server/api.md b/examples/server/api.md
index 3ec9a1da8f..bb42523661 100644
--- a/examples/server/api.md
+++ b/examples/server/api.md
@@ -37,6 +37,7 @@ Current generation-related endpoints include:
 
 - `POST /sdapi/v1/txt2img`
 - `POST /sdapi/v1/img2img`
+- `GET /sdapi/v1/progress`
 - `GET /sdapi/v1/loras`
 - `GET /sdapi/v1/upscalers`
 - `GET /sdapi/v1/latent-upscale-modes`
@@ -56,6 +57,294 @@ Current endpoints include:
 - `GET /sdcpp/v1/jobs/{id}`
 - `POST /sdcpp/v1/jobs/{id}/cancel`
 - `POST /sdcpp/v1/vid_gen`
+- `POST /sdcpp/v1/upscale`
+
+## Model management
+
+Model management is additive: existing generation request bodies, successful
+responses, job polling and cancellation remain unchanged. All three API families
+use the current loaded context. Starting with the existing model arguments
+preserves command-line preloading and its generation defaults; it does not
+require a recipe.
+
+Recipe defaults apply when a generation request omits the corresponding
+parameters. Image editing endpoints retain their existing input-image size
+inference when dimensions are omitted.
+
+Without a loaded model, generation endpoints return HTTP 503 with
+`{"error":"No model is loaded","code":"model_not_loaded"}`.
+While loading/unloading, endpoints that access the context return HTTP 503
+with `code: "model_loading"`. These errors retain the existing string-valued
+`error` field. Job polling/cancellation, catalog queries and model status remain
+available during loading. Model lists are empty when no model is loaded;
+`/v1/models` retains the `sd-cpp-local` ID when ready.
+
+| Method | Endpoint | Response / behavior |
+| --- | --- | --- |
+| GET | `/sdcpp/v1/recipes` | `{recipes: [...], errors: [...]}` |
+| GET | `/sdcpp/v1/model-files` | `{files: [{id, name, category, root, relative_path}], errors: [...]}` |
+| POST | `/sdcpp/v1/models/refresh` | Rescan recipes and files; return `{errors: [...]}` |
+| GET | `/sdcpp/v1/models/status` | Current state, recipe, selection, error, loading time and optional revision |
+| POST | `/sdcpp/v1/models/load` | Load the submitted recipe and selection; return status after completion |
+| POST | `/sdcpp/v1/models/unload` | Release the active context; return empty status |
+
+Load request:
+
+```json
+{
+  "recipe": "flux1-dev",
+  "models": {
+    "diffusion_model": "diffusion_models/flux1-dev.gguf",
+    "clip_l": "text_encoders/clip_l.safetensors",
+    "t5xxl": "text_encoders/t5xxl.safetensors",
+    "vae": "vae/ae.safetensors"
+  }
+}
+```
+
+File IDs are opaque strings returned by `model-files`. The first model root
+retains the existing relative IDs, such as `checkpoints/nested/model.gguf`.
+Additional roots use `@<root-id>/<relative-path>` so identical paths in different
+roots remain selectable. Root qualifiers derive from the canonical root path;
+adding unrelated roots or refreshing the catalog does not change them. Moving
+a root changes its qualifier. Do not construct IDs or use display names in load
+requests.
+
+The display `name` omits the physical category directory (including aliases such
+as `Stable-diffusion`). Duplicate display paths use the full absolute file path,
+with the source directories before the filename and no hash suffix.
+`root` is an opaque source identifier, and
+`relative_path` is the file path beneath that root. Duplicate canonical files
+appear only once, using the first root that discovers them. Categories depend
+on directories, never on filename suffixes such as `.vae.safetensors`.
+
+Repeated `--model-dir` arguments merge roots in order; absent arguments default
+to `./models`. User recipes override built-ins; the earliest root containing a
+valid user recipe wins across roots. Duplicate recipe IDs within one root are
+rejected. Directory aliases do not change recipe slot categories.
+
+LoRA lists retain relative `path` IDs for `loras/` in the first model root and
+for an explicit `--lora-model-dir`, and qualify other directories. The additive `display_path` field provides a readable
+relative path, or the full absolute file path when needed to distinguish duplicates.
+Continue sending `path` in structured LoRA requests. `--lora-model-dir` overrides
+automatic discovery across all model roots.
+Absolute paths, unknown files, wrong categories and
+paths escaping the configured root are rejected. Optional slots can be omitted
+or set to an empty string. Load/unload returns HTTP 409 (`model_busy`) while
+requests or queued jobs are pending. Invalid configurations return HTTP 400
+(`invalid_model_config`) before unloading the current context. An initialization
+failure returns HTTP 422 (`model_load_failed`) and leaves no active context.
+The load HTTP request waits for completion; a separate client can poll status
+while it runs. Refreshing the catalog never modifies an already loaded context.
+
+Status uses `state: empty | loading | ready | error`, `recipe` (ID or null),
+`models` (slot-to-file-ID mapping), and `error` (string or null). A monotonically
+increasing `revision` identifies runtime transitions within this server process.
+Command-line preloading keeps `origin: command_line` and `recipe: null` for
+compatibility, and adds `runtime_recipe`: a read-only `@command-line` recipe
+with all feature declarations enabled. It is only available while that context
+is loaded and cannot be submitted to `models/load`. The UI selects it by default
+and uses command-line generation defaults, not defaults from a catalog recipe.
+Capabilities also include this status as an additive `runtime` field.
+
+`load_elapsed_seconds` is the server-measured duration in seconds (a number or
+null). It updates while a model is loading and freezes on success or failure.
+Timing starts after selection validation and includes releasing the previous
+model. It resets for each load and is null before any load and during/after
+unloading. Command-line preloading reports its completed initialization time.
+Poll `/sdcpp/v1/models/status` to display elapsed time, including while loading;
+timer updates alone do not change `revision`.
+
+### Recipe feature declarations
+
+Recipes may declare `features` and `disabled_features` arrays. The server starts
+with shared defaults, enables features declared by component slots and `features`,
+then applies `disabled_features` last. A disabled feature always wins. Unknown
+names, non-array values and duplicate names within an array are rejected.
+Both fields are optional; existing recipes remain valid.
+
+```json
+{
+  "features": ["flow_shift", "distilled_guidance"],
+  "disabled_features": ["clip_skip", "lora"]
+}
+```
+
+Default-enabled features are `img2img`, `mask_image`, `lora`, `vae_tiling`,
+`cache`, `cache_dit` and `preview`. All other features below default to false:
+
+| Feature | Controls |
+| --- | --- |
+| `flow_shift` | Flow shift, where used by the selected scheduler |
+| `distilled_guidance` | Distilled guidance embedding |
+| `shifted_timestep` | SDXL timestep shifting |
+| `img_cfg` | Image CFG |
+| `slg` | Skip-layer guidance fields |
+| `clip_skip` | CLIP Skip |
+| `img2img`, `mask_image` | Image initialization/strength and masks |
+| `ref_images` | Reference image inputs |
+| `controlnet` | Control image and ControlNet strength |
+| `control_image` | Control image without a separate ControlNet (FLUX Canny/Depth) |
+| `ip_adapter` | IP-Adapter image and strength |
+| `init_image`, `end_image` | Video start/end frames |
+| `control_frames`, `vace` | Video control frames and VACE strength |
+| `high_noise` | High-noise pass, MoE boundary and high-noise LoRAs |
+| `lora`, `vae_tiling` | LoRA overrides and VAE tiling |
+| `cache`, `cache_unet`, `cache_dit` | Cache panel and architecture-specific algorithms |
+| `preview`, `preview_tae` | Preview panel and TAE preview option |
+
+A `control_net` slot enables `controlnet`, `ip_adapter` enables `ip_adapter`,
+`high_noise_diffusion_model` enables `high_noise`, and `tae`/`taesd` enables `preview_tae`, unless explicitly disabled.
+ControlNet controls are shown when previewing a supporting recipe, even before
+a model file is selected. High-noise and TAE controls additionally require their
+component to be selected. Runtime capabilities require the component to be loaded.
+PuLID disables SLG while selected.
+Image-only and video-only inputs are filtered by the active generation mode.
+Eta depends on the selected sampler; SCM fields only apply to DBCache,
+TaylorSeer and Cache-DIT. Empty panels and advanced sections are hidden.
+
+`GET /sdcpp/v1/recipes` adds a resolved boolean `effective_features` object to
+each recipe. This is response-only metadata, like `source` and slot `categories`;
+omit it when writing a user recipe file. The defaults are maintained by the
+server, not duplicated by the frontend.
+
+`GET /sdcpp/v1/capabilities` adds `generation_features` for the loaded context,
+derived from the active recipe and component availability without inferring
+features from model-family names. When no model is loaded, every feature is
+present and false. The UI can still preview a selected recipe using its
+`effective_features` before loading.
+Existing `features` / `features_by_mode` keys are retained and their model-specific
+values reflect these declarations. A loaded recipe is snapshotted: refreshing
+recipe files does not change the active context's capabilities until reloading.
+Command-line preloading uses the runtime-only all-features recipe instead of model-family inference
+and component paths; variant-specific recipe declarations are more precise.
+
+These declarations govern the Web UI and advertised capabilities, not API
+request validation. Existing generation APIs continue accepting their existing
+fields. The Web UI omits unsupported/hidden fields and ignores hidden image
+inputs when building requests, so switching recipes cannot submit stale options.
+
+### User recipe format
+
+Contributions to the built-in catalog must follow the
+[built-in recipe policy](recipes/README.md). Personal user recipes may define
+workflow-specific presets without adding them to the built-in catalog.
+
+Each UTF-8 JSON file in `<model-dir>/recipes/` defines one complete recipe.
+The maximum recipe file size is 1 MiB. Unknown fields, invalid types and
+duplicate slots are errors. Built-ins in `examples/server/recipes/*.json` use
+the same one-recipe-per-file schema:
+
+```json
+{
+  "schema_version": 1,
+  "id": "my-flux",
+  "name": "My FLUX",
+  "description": "FLUX.1 dev",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-dev",
+  "slots": [
+    {"id": "diffusion_model", "required": true},
+    {"id": "clip_l", "required": true},
+    {"id": "t5xxl", "required": true},
+    {"id": "vae", "required": false},
+    {"id": "tae", "required": false}
+  ],
+  "required_any": [["vae", "tae"]],
+  "defaults": {"width": 1024, "height": 1024, "cfg_scale": 1.0, "distilled_guidance": 3.5, "sampling_method": "euler", "steps": 20}
+}
+```
+
+IDs contain 1–128 ASCII letters, digits, dots, underscores or hyphens.
+`name`, `schema_version: 1` and `slots` are required; `description` and
+`model_url`, `required_any`, `defaults`, `context`, `features` and `disabled_features` are optional. Each slot requires only `id` and boolean `required`.
+The server supplies the default label and search directories for each slot.
+An optional nonempty `label` overrides the display name, for example
+`{"id": "diffusion_model", "label": "Low-noise diffusion model", "required": true}`.
+Legacy `category` fields remain accepted, but must match one of the slot's
+supported directories; they cannot redefine or restrict its search directories.
+At least one required `model` or `diffusion_model` slot is necessary.
+
+`model_url` is an optional HTTPS Hugging Face model repository link, shown beside
+the recipe description. Built-ins link to the model's author/publisher, not repacks.
+`required_any` contains groups of declared slot IDs. At least one model in each
+group must be selected. For example, `"required_any": [["vae", "tae"]]` with both
+slots marked `required: false` allows either the full VAE or a compatible TAE.
+Selecting both is allowed; the current server uses TAE for encoding/decoding when
+TAE is selected. Checkpoint recipes with a bundled VAE do not need this group.
+Use complete, model-compatible TAE weights; decoder-only preview weights cannot
+replace a full VAE. Pixel-space models and unsupported latent formats have no
+TAE slot.
+
+| Slot IDs | Search directories |
+| --- | --- |
+| `model` | `checkpoints` |
+| `diffusion_model`, `high_noise_diffusion_model`, `uncond_diffusion_model` | `diffusion_models` |
+| `clip_l`, `clip_g`, `t5xxl`, `llm`, `audio_encoder`, `embeddings_connectors` | `text_encoders` |
+| `clip_vision`, `llm_vision` | `clip_vision` |
+| `vae`, `audio_vae` | `vae` |
+| `tae` (legacy alias: `taesd`) | `vae` or `vae_approx` |
+| `tokenizer` | `tokenizers` |
+| `control_net` | `controlnet` |
+| `ip_adapter` | `ipadapter` |
+| `motion_module` | `motion_modules` |
+| `photo_maker` | `photomaker` |
+| `pulid_weights` | `pulid` |
+
+Built-in recipes use `tae` for the Tiny AutoEncoder slot. TAE selections can use files from either `vae/` or the legacy
+`vae_approx/` directory. Existing recipes and load requests using `taesd`
+remain supported. Do not specify both `tae` and `taesd` in the same recipe
+or load request.
+
+Model files can use `.gguf`, `.safetensors`, `.sft`, `.ckpt`, `.pt`, `.pth`
+or `.bin` extensions. The tokenizer category lists `.json` files only.
+Custom recipes compose components supported by the backend; they do not add
+new model architectures.
+
+Optional `context` settings apply to each model load and override the matching
+startup setting. Supported fields are `vae_format` (`auto`, `flux`, `sd3`,
+`flux2`, or `wan`) and `model_args` (a string in the same format as
+`--model-args`; replaces the startup string). Other startup settings remain
+unchanged. Switching to a recipe that omits these fields restores the startup
+values. For example, PiD uses `{"vae_format": "flux"}` and Qwen Image Edit
+2511 uses `{"model_args": "qwen_image_zero_cond_t=true"}`.
+
+Supported defaults:
+
+- Integer `width`/`height` (64–4096), `steps` (1–1000), `video_frames`
+  (1–4096), `fps` (1–240), `shifted_timestep` (0–1000), and
+  `high_noise_steps` (1–1000 or -1 for automatic).
+- Numeric `cfg_scale`, `img_cfg_scale`, `distilled_guidance`, `flow_shift`,
+  `high_noise_cfg_scale`, and `high_noise_flow_shift` (0–1000).
+- Numeric `strength` (0–1), used for image-to-image denoising, and
+  `moe_boundary` (0–1), used for automatic Wan MoE expert switching.
+- `custom_sigmas`: an empty array or 2–1001 finite, nonnegative numbers.
+  A nonempty array overrides the generated schedule; the sigma count minus one
+  determines the actual step count. Include the final zero. An empty array uses
+  the generated schedule. Loaded recipe sigmas apply to generation requests
+  unless `sample_params.custom_sigmas` overrides them; switching recipes restores
+  that recipe's schedule or the startup default.
+- Supported `sampling_method`/`scheduler` and
+  `high_noise_sampling_method`/`high_noise_scheduler` names.
+
+Other generation parameters continue to use startup defaults unless explicitly
+supplied by the generation request. Built-in Wan recipes provide video defaults;
+use `/sdcpp/v1/vid_gen` after loading a video model.
+LoRAs are merged from `loras/` and `Lora/` in every model root unless
+`--lora-model-dir` is supplied.
+
+Recipe responses fill in each slot's `label`, `category` (the legacy value,
+if supplied, or the first search directory), and `categories` (the complete
+server-defined list of search directories). Clients should filter model files
+using `categories`; `category` remains available for older clients.
+For example, `{"id": "tae", "required": false}` expands to
+`{"id": "tae", "required": false, "label": "TAE", "category": "vae", "categories": ["vae", "vae_approx"]}`.
+
+Recipe responses also add `source: builtin | user` and, for user recipes, `file`.
+Do not include `source`, `file`, or slot `categories` in a recipe file. A valid user recipe
+replaces a built-in with the same ID in full; inheritance/partial merging is
+not supported. Invalid recipes are skipped. Multiple valid user files sharing
+an ID are all skipped, preserving any built-in with that ID. Catalog errors
+are returned as `{file, message}` entries.
 
 ## `sd_cpp_extra_args`
 
@@ -148,6 +437,23 @@ Native extension fields:
 
 - any `sdcpp API` fields embedded through `sd_cpp_extra_args` inside `prompt`
 
+Uploaded images are decoded at their original dimensions. The first decoded
+image establishes the generation dimensions if `size` is omitted. Input
+geometry follows `image_preprocess`: references preserve their dimensions by
+default, while init and mask use the generation canvas preset.
+
+When the request starts without preconfigured reference images, the first uploaded image becomes the img2img init image if the loaded model does not support reference-image conditioning, the final reference list is empty, a mask is present, or an explicit numeric `strength` override is provided. Otherwise, only the final reference list is used for image conditioning, without initializing the img2img latent. Requests with preconfigured references retain their existing init-image selection.
+
+Routing uses the reference list and mask after applying `sd_cpp_extra_args`. Both `ref_images: []` and `ref_images: null` clear the reference list; `mask_image: null` clears an uploaded mask. An explicit `init_image` in `sd_cpp_extra_args`, including `null`, takes precedence over the implicit init-image selection.
+
+Reference encoding then follows model presets and `ref_image_args`. To skip
+input geometry for references and disable resizing before VAE encoding, include
+this in `prompt`:
+
+```text
+edit this image <sd_cpp_extra_args>{"image_preprocess":"target=ref,mode=none","ref_image_args":"resize_before_vae=false"}</sd_cpp_extra_args>
+```
+
 Response fields:
 
 | Field | Type | Notes |
@@ -264,6 +570,26 @@ Response fields:
 | `parameters` | `object` | Echo of the parsed outer request body |
 | `info` | `string` | Currently empty string |
 
+#### `GET /sdapi/v1/progress`
+
+Poll this endpoint while a synchronous SDAPI generation request is running.
+An optional `id_task` query parameter selects the task with the same `id_task`
+provided in the generation request (default: `"sdapi"`). Set
+`skip_current_image=true` or `1` to omit the preview image.
+
+`current_image` contains the latest base64 JPEG latent projection, or `null`
+when unavailable. `state.sampling_step` is the positive logical step and
+`state.sampling_steps` is the actual step count for the current sampling pass,
+including schedule and img2img strength adjustments. `state.job_no` is the
+zero-based pass index; `state.job_count` includes one pass per batch image and
+an additional pass per image when highres fix is enabled. `progress` weights
+these passes equally. Sampling progress can reach `1` before final decoding
+finishes; `eta_relative` is currently always `0`.
+
+When no SDAPI task is active, or `id_task` does not match, the endpoint returns
+`progress=0`, `state.job=""`, zero job/step counts, and `current_image=null`.
+Completed and failed requests do not retain an active preview.
+
 #### Discovery / Compatibility Endpoints
 
 Currently exposed:
@@ -374,7 +700,7 @@ Common job shape:
   "id": "job_01HTXYZABC",
   "kind": "img_gen",
   "status": "queued",
-  "created": 1775401200,
+  "created": 1775401200000,
   "started": null,
   "completed": null,
   "queue_position": 2,
@@ -396,6 +722,25 @@ Field types:
 | `queue_position` | `integer` |
 | `result` | `object \| null` |
 | `error` | `object \| null` |
+| `preview` | `object \| null` |
+
+`created`, `started`, and `completed` are integer Unix timestamps in
+**milliseconds**. Subtract `started` from `completed` and divide by 1000 to get
+the elapsed generation time in seconds, including output encoding and excluding
+queue time.
+
+`preview` sub-fields:
+
+| Field | Type | Notes |
+| --- | --- | --- |
+| `preview.pass` | `integer` | Sampling pass number, starting at 1 within the job |
+| `preview.step` | `integer` | Positive logical sample step within this pass |
+| `preview.total_steps` | `integer` | Actual sample steps in this pass, after schedule and strength adjustments |
+| `preview.b64_json` | `string` | Base64-encoded PNG of the current preview frame |
+
+The preview is updated in place as sampling progresses; poll `GET /sdcpp/v1/jobs/{id}` to retrieve the latest frame. Only the most recent preview is retained.
+
+`step / total_steps` describes the current sampling pass, not overall job completion. A new batch item, high/low-noise stage, or highres pass increments `pass` and restarts `step`. Samplers with multiple denoiser evaluations per logical step may update a preview more than once at the same step. The frame and its pass/step metadata are published together.
 
 ### Endpoints
 
@@ -421,7 +766,9 @@ Top-level fields:
 | `samplers` | `array<string>` | Available sampling methods |
 | `schedulers` | `array<string>` | Available schedulers |
 | `loras` | `array<object>` | Available LoRA entries |
-| `upscalers` | `array<object>` | Available model-backed highres upscalers |
+| `upscalers` | `array<object>` | Available highres upscalers, built-in and model-backed |
+| `upscale` | `boolean` | Whether a compatible RGB ESRGAN model is available for `POST /sdcpp/v1/upscale` |
+| `preview_modes` | `array<string>` | Available preview modes, e.g. `["none", "proj", "tae", "vae"]` |
 | `limits` | `object` | Shared queue and size limits |
 
 `model`
@@ -457,12 +804,15 @@ Shared nested fields:
 | --- | --- |
 | `loras[].name` | `string` |
 | `loras[].path` | `string` |
+| `loras[].display_path` | `string` (display only) |
 
 `upscalers`
 
 | Field | Type | Notes |
 | --- | --- | --- |
 | `upscalers[].name` | `string` | Built-in name or model stem; use this value in `hires.upscaler` |
+| `upscalers[].model` | `boolean` | True for a model-backed upscaler, false for a built-in scaling filter |
+| `upscalers[].image_upscale` | `boolean` | Whether this model can be selected by `POST /sdcpp/v1/upscale`; false for latent upscalers and built-in filters |
 
 Built-in entries include `None`, `Lanczos`, `Nearest`, `Latent`, `Latent (nearest)`, `Latent (nearest-exact)`, `Latent (antialiased)`, `Latent (bicubic)`, and `Latent (bicubic antialiased)`. Model-backed entries are scanned from the top level of `--hires-upscalers-dir`; subdirectories are not scanned.
 
@@ -476,6 +826,8 @@ Built-in entries include `None`, `Lanczos`, `Nearest`, `Latent`, `Latent (neares
 | `limits.max_height` | `integer` |
 | `limits.max_batch_count` | `integer` |
 | `limits.max_queue_size` | `integer` |
+| `limits.max_upscale_width` | `integer` |
+| `limits.max_upscale_height` | `integer` |
 
 Shared default fields used by both `img_gen` and `vid_gen`:
 
@@ -505,11 +857,11 @@ Shared default fields used by both `img_gen` and `vid_gen`:
 | `vae_tiling_params` | `object` |
 | `vae_tiling_params.enabled` | `boolean` |
 | `vae_tiling_params.temporal_tiling` | `boolean` |
-| `vae_tiling_params.tile_size_x` | `integer` |
-| `vae_tiling_params.tile_size_y` | `integer` |
+| `vae_tiling_params.tile_size_w` | `integer` |
+| `vae_tiling_params.tile_size_h` | `integer` |
 | `vae_tiling_params.target_overlap` | `number` |
-| `vae_tiling_params.rel_size_x` | `number` |
-| `vae_tiling_params.rel_size_y` | `number` |
+| `vae_tiling_params.rel_size_w` | `number` |
+| `vae_tiling_params.rel_size_h` | `number` |
 | `vae_tiling_params.extra_tiling_args` | `string` |
 | `cache_mode` | `string` |
 | `cache_option` | `string` |
@@ -518,6 +870,8 @@ Shared default fields used by both `img_gen` and `vid_gen`:
 | `output_format` | `string` |
 | `output_compression` | `integer` |
 
+`vae_tiling_params.tile_size_w` and `tile_size_h` are in **image pixels**, with `0` selecting the 256-pixel default. Both encode and decode use these sizes without an encoding multiplier. Positive `rel_size_w`/`rel_size_h` values override the corresponding absolute size: values up to 1 are dimension fractions, and values greater than 1 are target tile counts. Set `enabled` to use spatial tiling. Sizes are aligned down to the VAE scale factor and capped at the input dimensions; explicit sizes below the minimum supported tile size are rejected. These fields previously used latent units; see [VAE tiling](../../docs/performance.md#use-vae-tiling-to-reduce-encode-and-decode-memory-usage) for migration and OOM retry behavior.
+
 `vae_tiling_params.extra_tiling_args` accepts a key=value list. Supported video VAEs accept `temporal_tile_frames` (alias `temporal_tile_size`, default `4`) and `temporal_tile_overlap` (default `1`).
 LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEHV use overlap blending. MiniMax H3 keeps its model-specific fixed temporal windows because its latent-to-frame mapping is non-linear.
 
@@ -526,7 +880,7 @@ LTX and Wan preserve causal state between temporal tiles. Hunyuan Video and TAEH
 | Field | Type |
 | --- | --- |
 | `batch_count` | `integer` |
-| `auto_resize_ref_image` | `boolean` |
+| `ref_image_args` | `string` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
 | `ip_adapter_strength` | `number` |
@@ -577,6 +931,7 @@ Fields returned in `features_by_mode.img_gen`:
 - `cache`
 - `cancel_queued`
 - `cancel_generating`
+- `preview`
 
 Fields returned in `features_by_mode.vid_gen`:
 
@@ -589,6 +944,7 @@ Fields returned in `features_by_mode.vid_gen`:
 - `cache`
 - `cancel_queued`
 - `cancel_generating`
+- `preview`
 
 #### `POST /sdcpp/v1/img_gen`
 
@@ -603,7 +959,7 @@ Example response:
   "id": "job_01HTXYZABC",
   "kind": "img_gen",
   "status": "queued",
-  "created": 1775401200,
+  "created": 1775401200000,
   "poll_url": "/sdcpp/v1/jobs/job_01HTXYZABC"
 }
 ```
@@ -628,6 +984,52 @@ Typical status codes:
 - `404 Not Found`
 - `410 Gone`
 
+#### `POST /sdcpp/v1/upscale`
+
+Runs one RGB ESRGAN upscaler over an image, with no generation involved. Latent upscaler models remain available for hires generation but cannot be used here.
+
+This is the HTTP equivalent of `sd-cli -M upscale`: no diffusion model, text
+encoder or sampling is used, so it is fast enough to answer synchronously and
+does not create a job.
+
+Request fields:
+
+| Field | Type | Notes |
+| --- | --- | --- |
+| `image` | `string` | Required. Base64 or data URL image |
+| `upscaler` | `string` | A name from `upscalers` with `image_upscale: true`; the first compatible entry when omitted |
+| `repeats` | `integer` | Run the upscaler this many times, 1 to 4 (default `1`) |
+| `tile_size` | `integer` | Tile size, defaulting to the server's `--upscale-tile-size` |
+| `output_format` | `string` | `png`, `jpeg`, or `webp` when built with WebP support (default `png`); unsupported formats return 400 |
+| `output_compression` | `integer` | Range is clamped to `0..100` |
+
+Response fields:
+
+| Field | Type | Notes |
+| --- | --- | --- |
+| `images` | `array<object>` | One image |
+| `images[].index` | `integer` | |
+| `images[].b64_json` | `string` | Base64-encoded image bytes |
+| `upscaler` | `string` | The upscaler actually used |
+| `scale` | `integer` | The model's scale factor |
+| `repeats` | `integer` | How many times it was run |
+| `width` | `integer` | Result width |
+| `height` | `integer` | Result height |
+| `output_format` | `string` | Final encoded image format |
+
+Typical status codes:
+
+- `200 OK`
+- `400 Bad Request` (invalid request, unsupported output format, unreadable image, incompatible upscaler, or output dimensions exceeding the limit)
+- `500 Internal Server Error`
+
+Notes:
+
+- Final output dimensions, including all repeats, must not exceed 8192 pixels on either axis (`limits.max_upscale_width` and `limits.max_upscale_height`). Requests exceeding this bound are rejected before upscaling.
+- The upscaler models are three-channel; alpha is not preserved.
+- The request holds the generation context lock, so an upscale and a
+  generation never run on the device at the same time.
+
 #### `POST /sdcpp/v1/jobs/{id}/cancel`
 
 Attempts to cancel an accepted job.
@@ -653,7 +1055,7 @@ Example:
   "strength": 0.75,
   "seed": -1,
   "batch_count": 1,
-  "auto_resize_ref_image": true,
+  "ref_image_args": "",
   "increase_ref_index": false,
   "control_strength": 0.9,
   "ip_adapter_strength": 1.0,
@@ -702,11 +1104,11 @@ Example:
   "vae_tiling_params": {
     "enabled": false,
     "temporal_tiling": false,
-    "tile_size_x": 0,
-    "tile_size_y": 0,
+    "tile_size_w": 0,
+    "tile_size_h": 0,
     "target_overlap": 0.5,
-    "rel_size_x": 0.0,
-    "rel_size_y": 0.0,
+    "rel_size_w": 0.0,
+    "rel_size_h": 0.0,
     "extra_tiling_args": ""
   },
 
@@ -715,6 +1117,9 @@ Example:
   "scm_mask": "",
   "scm_policy_dynamic": true,
 
+  "preview": "none",
+  "preview_interval": 1,
+
   "output_format": "png",
   "output_compression": 100
 }
@@ -728,6 +1133,17 @@ Example:
 
 ### Image Encoding Rules
 
+Native image/video requests and SDAPI accept `image_preprocess` as a rule string
+or array of rule strings. OpenAI-compatible requests can supply it in
+`sd_cpp_extra_args`. See [Image preprocessing](../../docs/image_preprocessing.md)
+for one-time input geometry, native-resolution decoding, mask alignment, and
+`canny=true` for edge detection on any supported image input.
+
+Image generation also accepts `ref_image_args` as a string (for example,
+`"resize_before_vae=false"`) in native and SDAPI requests, or through
+`sd_cpp_extra_args` in OpenAI-compatible requests. It controls downstream
+reference encoding and is independent of input geometry rules.
+
 Any image field accepts:
 
 - a raw base64 string, or
@@ -735,12 +1151,15 @@ Any image field accepts:
 
 Channel expectations:
 
-- `init_image`: 3 channels
-- `ref_images[]`: 3 channels
+- `init_image`: native channels (3 or 4); alpha is preserved and applied per model
+- `ref_images[]`: native channels (3 or 4); alpha is preserved and applied per model
 - `control_image`: 3 channels
 - `ip_adapter_image`: 3 channels
 - `mask_image`: 1 channel
 
+Models that support RGBA (e.g. Qwen-Image 2.1) use the alpha channel of `init_image`
+and `ref_images[]`. RGB-only models drop it, so sending RGBA is safe for every model.
+
 If omitted or null:
 
 - single-image fields map to an empty `sd_image_t`
@@ -760,11 +1179,14 @@ Top-level scalar fields:
 | `strength` | `number` |
 | `seed` | `integer` |
 | `batch_count` | `integer` |
-| `auto_resize_ref_image` | `boolean` |
+| `ref_image_args` | `string` |
+| `image_preprocess` | `string \| array<string>` |
 | `increase_ref_index` | `boolean` |
 | `control_strength` | `number` |
 | `ip_adapter_strength` | `number` |
 | `embed_image_metadata` | `boolean` |
+| `preview` | `string` |
+| `preview_interval` | `integer` |
 
 Image fields:
 
@@ -820,11 +1242,11 @@ Other native fields:
 | `vae_tiling_params` | `object` |
 | `vae_tiling_params.enabled` | `boolean` |
 | `vae_tiling_params.temporal_tiling` | `boolean` |
-| `vae_tiling_params.tile_size_x` | `integer` |
-| `vae_tiling_params.tile_size_y` | `integer` |
+| `vae_tiling_params.tile_size_w` | `integer` |
+| `vae_tiling_params.tile_size_h` | `integer` |
 | `vae_tiling_params.target_overlap` | `number` |
-| `vae_tiling_params.rel_size_x` | `number` |
-| `vae_tiling_params.rel_size_y` | `number` |
+| `vae_tiling_params.rel_size_w` | `number` |
+| `vae_tiling_params.rel_size_h` | `number` |
 | `vae_tiling_params.extra_tiling_args` | `string` |
 | `cache_mode` | `string` |
 | `cache_option` | `string` |
@@ -852,6 +1274,10 @@ When omitted, backend defaults apply to these fields:
 - `sample_params.flow_shift`
 - `sample_params.guidance.img_cfg`
 
+### Preview Interval Semantics
+
+`preview_interval` controls the period (in sample steps) at which preview frames are generated. The default is `1` (every step). Any non-positive values will be clamped to `1`. Note that if `preview` is set to `"none"`, no previews are produced regardless of the interval.
+
 ### Completion Result
 
 Example completed job:
@@ -861,9 +1287,9 @@ Example completed job:
   "id": "job_01HTXYZABC",
   "kind": "img_gen",
   "status": "completed",
-  "created": 1775401200,
-  "started": 1775401203,
-  "completed": 1775401215,
+  "created": 1775401200000,
+  "started": 1775401203000,
+  "completed": 1775401215237,
   "queue_position": 0,
   "result": {
     "output_format": "png",
@@ -887,9 +1313,9 @@ Example failed job:
   "id": "job_01HTXYZABC",
   "kind": "img_gen",
   "status": "failed",
-  "created": 1775401200,
-  "started": 1775401203,
-  "completed": 1775401204,
+  "created": 1775401200000,
+  "started": 1775401203000,
+  "completed": 1775401204000,
   "queue_position": 0,
   "result": null,
   "error": {
@@ -908,9 +1334,9 @@ Example cancelled job:
   "id": "job_01HTXYZABC",
   "kind": "img_gen",
   "status": "cancelled",
-  "created": 1775401200,
+  "created": 1775401200000,
   "started": null,
-  "completed": 1775401202,
+  "completed": 1775401202000,
   "queue_position": 0,
   "result": null,
   "error": {
@@ -946,7 +1372,7 @@ Example response:
   "id": "job_01HTXYZVID",
   "kind": "vid_gen",
   "status": "queued",
-  "created": 1775401200,
+  "created": 1775401200000,
   "poll_url": "/sdcpp/v1/jobs/job_01HTXYZVID"
 }
 ```
@@ -966,7 +1392,8 @@ Response fields:
 Compared with `img_gen`, the `vid_gen` request body:
 
 - `vid_gen` is a single video sequence job, so `batch_count` is not part of the request schema
-- `ref_images`, `mask_image`, `control_image`, `control_strength`, `ip_adapter_image`, `ip_adapter_strength`, and `embed_image_metadata` are not part of the request schema
+- `mask_image`, `control_image`, `control_strength`, `ip_adapter_image`, `ip_adapter_strength`, and `embed_image_metadata` are not part of the request schema
+- `ref_images` is accepted for MiniMax-H3 Ref2VA conditioning; other video model families currently ignore it
 - `vid_gen` adds `end_image`, `control_frames`, `high_noise_sample_params`, `video_frames`, `fps`, `moe_boundary`, and `vace_strength`
 
 Example:
@@ -987,6 +1414,7 @@ Example:
 
   "init_image": null,
   "end_image": null,
+  "ref_images": [],
   "control_frames": [],
 
   "sample_params": {
@@ -1035,11 +1463,11 @@ Example:
   "vae_tiling_params": {
     "enabled": false,
     "temporal_tiling": false,
-    "tile_size_x": 0,
-    "tile_size_y": 0,
+    "tile_size_w": 0,
+    "tile_size_h": 0,
     "target_overlap": 0.5,
-    "rel_size_x": 0.0,
-    "rel_size_y": 0.0,
+    "rel_size_w": 0.0,
+    "rel_size_h": 0.0,
     "extra_tiling_args": ""
   },
 
@@ -1048,11 +1476,25 @@ Example:
   "scm_mask": "",
   "scm_policy_dynamic": true,
 
+  "preview": "none",
+  "preview_interval": 1,
+
   "output_format": "webm",
   "output_compression": 100
 }
 ```
 
+### Reference Image Rules
+
+- `ref_images` contains reference images for MiniMax-H3 Ref2VA conditioning.
+- Images retain request order and correspond to `<Picture 1>`, `<Picture 2>`, and so on in the prompt.
+- For MiniMax-H3, non-empty `ref_images` cannot be combined with `init_image` or `end_image`. A conflicting API request fails during generation; the WebUI checks this before submission.
+- MiniMax-H3 does not support `control_frames`; leave that array empty.
+- Other video model families currently ignore `ref_images`.
+- `features_by_mode.vid_gen` does not currently advertise `ref_images`; its absence is not an indication that MiniMax-H3 reference images are unsupported.
+
+See [MiniMax-H3 reference conditioning](../../docs/minimax_h3.md#reference-to-audio-video-conditioning) for model requirements and prompt examples.
+
 ### LoRA Rules
 
 - The server only accepts explicit LoRA entries from the `lora` field.
@@ -1070,6 +1512,7 @@ Channel expectations:
 
 - `init_image`: 3 channels
 - `end_image`: 3 channels
+- `ref_images[]`: decoded with native channels, then converted to RGB by MiniMax-H3
 - `control_frames[]`: 3 channels
 
 Frame ordering rules:
@@ -1099,6 +1542,8 @@ Top-level scalar fields:
 | `fps` | `integer` |
 | `moe_boundary` | `number` |
 | `vace_strength` | `number` |
+| `preview` | `string` |
+| `preview_interval` | `integer` |
 
 Image and frame fields:
 
@@ -1106,6 +1551,7 @@ Image and frame fields:
 | --- | --- |
 | `init_image` | `string \| null` |
 | `end_image` | `string \| null` |
+| `ref_images` | `array<string>` |
 | `control_frames` | `array<string>` |
 
 LoRA fields:
@@ -1160,11 +1606,11 @@ Other native fields:
 | `vae_tiling_params` | `object` |
 | `vae_tiling_params.enabled` | `boolean` |
 | `vae_tiling_params.temporal_tiling` | `boolean` |
-| `vae_tiling_params.tile_size_x` | `integer` |
-| `vae_tiling_params.tile_size_y` | `integer` |
+| `vae_tiling_params.tile_size_w` | `integer` |
+| `vae_tiling_params.tile_size_h` | `integer` |
 | `vae_tiling_params.target_overlap` | `number` |
-| `vae_tiling_params.rel_size_x` | `number` |
-| `vae_tiling_params.rel_size_y` | `number` |
+| `vae_tiling_params.rel_size_w` | `number` |
+| `vae_tiling_params.rel_size_h` | `number` |
 | `vae_tiling_params.extra_tiling_args` | `string` |
 | `cache_mode` | `string` |
 | `cache_option` | `string` |
@@ -1254,9 +1700,9 @@ Example completed job:
   "id": "job_01HTXYZVID",
   "kind": "vid_gen",
   "status": "completed",
-  "created": 1775401200,
-  "started": 1775401203,
-  "completed": 1775401215,
+  "created": 1775401200000,
+  "started": 1775401203000,
+  "completed": 1775401215237,
   "queue_position": 0,
   "result": {
     "output_format": "webm",
@@ -1280,9 +1726,9 @@ Example failed job:
   "id": "job_01HTXYZVID",
   "kind": "vid_gen",
   "status": "failed",
-  "created": 1775401200,
-  "started": 1775401203,
-  "completed": 1775401204,
+  "created": 1775401200000,
+  "started": 1775401203000,
+  "completed": 1775401204000,
   "queue_position": 0,
   "result": null,
   "error": {
@@ -1301,9 +1747,9 @@ Example cancelled job:
   "id": "job_01HTXYZVID",
   "kind": "vid_gen",
   "status": "cancelled",
-  "created": 1775401200,
+  "created": 1775401200000,
   "started": null,
-  "completed": 1775401202,
+  "completed": 1775401202000,
   "queue_position": 0,
   "result": null,
   "error": {
diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp
index 287955a848..7f3c8a1bfc 100644
--- a/examples/server/async_jobs.cpp
+++ b/examples/server/async_jobs.cpp
@@ -10,6 +10,55 @@
 #include "common/media_io.h"
 #include "common/resource_owners.hpp"
 
+static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) {
+    (void)is_noisy;
+    if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) {
+        return;
+    }
+
+    AsyncGenerationJob* job      = static_cast<AsyncGenerationJob*>(data);
+    const sd_image_t& frame      = frames[0];
+    const sd_preview_info_t info = sd_get_preview_info();
+
+    auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG,
+                                              frame.data,
+                                              frame.width,
+                                              frame.height,
+                                              frame.channel,
+                                              "",
+                                              80);
+    if (image_bytes.empty()) {
+        return;
+    }
+
+    std::string b64 = base64_encode(image_bytes);
+    std::lock_guard<std::mutex> lock(job->preview_mutex);
+    job->preview_b64         = std::move(b64);
+    job->preview_step        = step < 0 ? -step : step;
+    job->preview_pass        = info.sample_pass;
+    job->preview_total_steps = info.total_steps;
+}
+
+static void clear_preview_callback() {
+    sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr);
+}
+
+static void set_preview_callback(AsyncGenerationJob& job) {
+    preview_t mode = str_to_preview(job.kind == AsyncJobKind::ImgGen
+                                        ? job.img_gen.preview_mode.c_str()
+                                        : job.vid_gen.preview_mode.c_str());
+    if (mode == PREVIEW_COUNT || mode == PREVIEW_NONE) {
+        return;
+    }
+    int interval = job.kind == AsyncJobKind::ImgGen
+                       ? job.img_gen.preview_interval
+                       : job.vid_gen.preview_interval;
+    if (interval <= 0) {
+        interval = 1;
+    }
+    sd_set_preview_callback(preview_callback, mode, interval, true, false, &job);
+}
+
 const char* async_job_kind_name(AsyncJobKind kind) {
     switch (kind) {
         case AsyncJobKind::ImgGen:
@@ -39,7 +88,7 @@ const char* async_job_status_name(AsyncJobStatus status) {
 }
 
 void purge_expired_jobs(AsyncJobManager& manager) {
-    const int64_t now = unix_timestamp_now();
+    const int64_t now = unix_timestamp_now_ms();
 
     for (auto it = manager.expired_jobs.begin(); it != manager.expired_jobs.end();) {
         if (it->second <= now) {
@@ -59,8 +108,8 @@ void purge_expired_jobs(AsyncJobManager& manager) {
         int64_t ttl_seconds = job->status == AsyncJobStatus::Completed
                                   ? manager.completed_ttl_seconds
                                   : manager.failed_ttl_seconds;
-        if (now - job->completed_at >= ttl_seconds) {
-            manager.expired_jobs[job->id] = now + std::max<int64_t>(ttl_seconds, 60);
+        if (now - job->completed_at >= ttl_seconds * 1000) {
+            manager.expired_jobs[job->id] = now + std::max<int64_t>(ttl_seconds, 60) * 1000;
             it                            = manager.jobs.erase(it);
         } else {
             ++it;
@@ -81,7 +130,7 @@ size_t count_pending_jobs(const AsyncJobManager& manager) {
 
 std::string make_async_job_id(AsyncJobManager& manager) {
     std::ostringstream oss;
-    oss << "job_" << std::hex << unix_timestamp_now() << "_" << std::setw(8)
+    oss << "job_" << std::hex << unix_timestamp_now_ms() << "_" << std::setw(8)
         << std::setfill('0') << manager.next_id++;
     return oss.str();
 }
@@ -94,7 +143,7 @@ bool cancel_queued_job(AsyncJobManager& manager, AsyncGenerationJob& job) {
 
     manager.queue.erase(new_end, manager.queue.end());
     job.status       = AsyncJobStatus::Cancelled;
-    job.completed_at = unix_timestamp_now();
+    job.completed_at = unix_timestamp_now_ms();
     job.result_images_b64.clear();
     job.result_media_b64.clear();
     job.result_media_mime_type.clear();
@@ -161,6 +210,18 @@ json make_async_job_json(const AsyncJobManager& manager, const AsyncGenerationJo
         result["error"]  = nullptr;
     }
 
+    if (job.status == AsyncJobStatus::Generating) {
+        std::lock_guard<std::mutex> lock(job.preview_mutex);
+        if (!job.preview_b64.empty()) {
+            result["preview"] = {
+                {"pass", job.preview_pass},
+                {"step", job.preview_step},
+                {"total_steps", job.preview_total_steps},
+                {"b64_json", job.preview_b64},
+            };
+        }
+    }
+
     return result;
 }
 
@@ -174,12 +235,14 @@ bool execute_img_gen_job(ServerRuntime& runtime,
 
     {
         std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
+        set_preview_callback(job);
         sd_image_t* raw_results = nullptr;
         int num_results         = 0;
         if (!generate_image(runtime.sd_ctx, &params, &raw_results, &num_results)) {
             raw_results = nullptr;
             num_results = 0;
         }
+        clear_preview_callback();
         results.adopt(raw_results, num_results);
     }
 
@@ -247,10 +310,12 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
 
     {
         std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
+        set_preview_callback(job);
         sd_image_t* raw_results = nullptr;
         if (!generate_video(runtime.sd_ctx, &params, &raw_results, &num_results, &generated_audio, &output_fps)) {
             raw_results = nullptr;
         }
+        clear_preview_callback();
         results.adopt(raw_results, num_results);
     }
 
@@ -308,7 +373,7 @@ void async_job_worker(ServerRuntime& runtime) {
 
             job             = it->second;
             job->status     = AsyncJobStatus::Generating;
-            job->started_at = unix_timestamp_now();
+            job->started_at = unix_timestamp_now_ms();
         }
 
         std::vector<std::string> output_images;
@@ -340,7 +405,7 @@ void async_job_worker(ServerRuntime& runtime) {
                 continue;
             }
 
-            job->completed_at = unix_timestamp_now();
+            job->completed_at = unix_timestamp_now_ms();
             if (ok) {
                 job->status                 = AsyncJobStatus::Completed;
                 job->result_images_b64      = std::move(output_images);
diff --git a/examples/server/async_jobs.h b/examples/server/async_jobs.h
index 89997a3b4a..049a9d73b3 100644
--- a/examples/server/async_jobs.h
+++ b/examples/server/async_jobs.h
@@ -32,7 +32,7 @@ struct AsyncGenerationJob {
     std::string id;
     AsyncJobKind kind     = AsyncJobKind::ImgGen;
     AsyncJobStatus status = AsyncJobStatus::Queued;
-    int64_t created_at    = unix_timestamp_now();
+    int64_t created_at    = unix_timestamp_now_ms();
     int64_t started_at    = 0;
     int64_t completed_at  = 0;
     ImgGenJobRequest img_gen;
@@ -44,6 +44,11 @@ struct AsyncGenerationJob {
     int result_fps         = 0;
     std::string error_code;
     std::string error_message;
+    mutable std::mutex preview_mutex;
+    std::string preview_b64;
+    int preview_pass          = 0;
+    int preview_step          = 0;
+    int preview_total_steps   = 0;
 };
 
 struct AsyncJobManager {
diff --git a/examples/server/frontend b/examples/server/frontend
index c4bce3d6b3..e36e34a2e2 160000
--- a/examples/server/frontend
+++ b/examples/server/frontend
@@ -1 +1 @@
-Subproject commit c4bce3d6b3f236614cca21014f076083b7270ba8
+Subproject commit e36e34a2e2204e3df654b16cadc112d8ce8f3174
diff --git a/examples/server/main.cpp b/examples/server/main.cpp
index 1f93e0dc7f..170877686b 100644
--- a/examples/server/main.cpp
+++ b/examples/server/main.cpp
@@ -10,6 +10,7 @@
 #include "async_jobs.h"
 #include "common/common.h"
 #include "common/resource_owners.hpp"
+#include "model_catalog.h"
 #include "routes.h"
 #include "runtime.h"
 
@@ -50,7 +51,7 @@ static void parse_args(int argc,
     const bool random_seed_requested = default_gen_params.seed < 0;
 
     if (!svr_params.resolve_and_validate() ||
-        !ctx_params.resolve_and_validate(IMG_GEN) ||
+        !ctx_params.resolve_and_validate(ctx_params.model_path.empty() && ctx_params.diffusion_model_path.empty() ? METADATA : IMG_GEN) ||
         !default_gen_params.resolve_and_validate(IMG_GEN,
                                                  ctx_params.lora_model_dir,
                                                  ctx_params.hires_upscalers_dir)) {
@@ -75,21 +76,28 @@ int main(int argc, const char** argv) {
     }
     SDSvrParams svr_params;
     SDContextParams ctx_params;
+    // Preserve the CLI default while disabling cwd scans for sd-server.
+    ctx_params.lora_model_dir.clear();
     SDGenerationParams default_gen_params;
-    parse_args(argc, argv, svr_params, ctx_params, default_gen_params);
 
     sd_set_log_callback(sd_log_cb, (void*)&svr_params);
+    parse_args(argc, argv, svr_params, ctx_params, default_gen_params);
 
     LOG_VERBOSE("version: %s", version_string().c_str());
     LOG_VERBOSE("%s", sd_get_system_info());
-    LOG_VERBOSE("%s", svr_params.to_string().c_str());
-    LOG_VERBOSE("%s", ctx_params.to_string().c_str());
-    LOG_VERBOSE("%s", default_gen_params.to_string().c_str());
+    LOG_DEBUG("%s", svr_params.to_string().c_str());
+    LOG_DEBUG("%s", ctx_params.to_string().c_str());
+    LOG_DEBUG("%s", default_gen_params.to_string().c_str());
 
     sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(false);
-    SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params));
+    ServerModels models;
+    if (!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) {
+        const auto started = std::chrono::steady_clock::now();
+        models.context.reset(new_sd_ctx(&sd_ctx_params));
+        models.status["load_elapsed_seconds"] = std::chrono::duration<double>(std::chrono::steady_clock::now() - started).count();
+    }
 
-    if (sd_ctx == nullptr) {
+    if ((!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) && !models.context) {
         LOG_ERROR("new_sd_ctx_t failed");
         return 1;
     }
@@ -102,7 +110,7 @@ int main(int argc, const char** argv) {
     std::mutex upscaler_mutex;
     AsyncJobManager async_job_manager;
     ServerRuntime runtime = {
-        sd_ctx.get(),
+        models.context.get(),
         &sd_ctx_mutex,
         &svr_params,
         &ctx_params,
@@ -112,8 +120,16 @@ int main(int argc, const char** argv) {
         &upscaler_cache,
         &upscaler_mutex,
         &async_job_manager,
+        &models,
     };
 
+    try {
+        initialize_server_models(runtime);
+    } catch (const std::exception& e) {
+        LOG_ERROR("model catalog initialization failed: %s", e.what());
+        return 1;
+    }
+
     std::thread async_worker(async_job_worker, std::ref(runtime));
 
     httplib::Server svr;
@@ -145,6 +161,7 @@ int main(int argc, const char** argv) {
     register_openai_api_endpoints(svr, runtime);
     register_sdapi_endpoints(svr, runtime);
     register_sdcpp_api_endpoints(svr, runtime);
+    register_model_endpoints(svr, runtime);
 
     LOG_INFO("listening on: http://%s:%d\n", svr_params.listen_ip.c_str(), svr_params.listen_port);
     svr.listen(svr_params.listen_ip, svr_params.listen_port);
diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp
new file mode 100644
index 0000000000..a3c59206d2
--- /dev/null
+++ b/examples/server/model_catalog.cpp
@@ -0,0 +1,664 @@
+#include "model_catalog.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cmath>
+#include <fstream>
+#include <iomanip>
+#include <set>
+#include <sstream>
+#include <stdexcept>
+#include <vector>
+
+#include "common/log.h"
+#include "recipes_builtin.h"
+
+namespace fs = std::filesystem;
+
+struct ModelSlot {
+    const char* id;
+    const char* label;
+    std::vector<std::string> categories;
+    std::string SDContextParams::*field;
+};
+
+static const ModelSlot k_model_slots[] = {
+    {"model", "Checkpoint", {"checkpoints"}, &SDContextParams::model_path},
+    {"diffusion_model", "Diffusion model", {"diffusion_models"}, &SDContextParams::diffusion_model_path},
+    {"high_noise_diffusion_model", "High-noise diffusion model", {"diffusion_models"}, &SDContextParams::high_noise_diffusion_model_path},
+    {"uncond_diffusion_model", "Unconditional diffusion model", {"diffusion_models"}, &SDContextParams::uncond_diffusion_model_path},
+    {"clip_l", "CLIP-L", {"text_encoders"}, &SDContextParams::clip_l_path},
+    {"clip_g", "CLIP-G", {"text_encoders"}, &SDContextParams::clip_g_path},
+    {"clip_vision", "CLIP Vision", {"clip_vision"}, &SDContextParams::clip_vision_path},
+    {"t5xxl", "T5", {"text_encoders"}, &SDContextParams::t5xxl_path},
+    {"llm", "LLM Text Encoder", {"text_encoders"}, &SDContextParams::llm_path},
+    {"llm_vision", "LLM Vision Encoder", {"clip_vision"}, &SDContextParams::llm_vision_path},
+    {"tokenizer", "Tokenizer", {"tokenizers"}, &SDContextParams::tokenizer},
+    {"vae", "VAE", {"vae"}, &SDContextParams::vae_path},
+    {"audio_vae", "Audio VAE", {"vae"}, &SDContextParams::audio_vae_path},
+    {"audio_encoder", "Audio Encoder", {"text_encoders"}, &SDContextParams::audio_encoder_path},
+    {"embeddings_connectors", "Embeddings Connectors", {"text_encoders"}, &SDContextParams::embeddings_connectors_path},
+    {"tae", "TAE", {"vae", "vae_approx"}, &SDContextParams::taesd_path},
+    {"taesd", "TAE", {"vae_approx", "vae"}, &SDContextParams::taesd_path},
+    {"control_net", "ControlNet", {"controlnet"}, &SDContextParams::control_net_path},
+    {"ip_adapter", "IP-Adapter", {"ipadapter"}, &SDContextParams::ip_adapter_path},
+    {"motion_module", "Motion Module", {"motion_modules"}, &SDContextParams::motion_module_path},
+    {"photo_maker", "PhotoMaker", {"photomaker"}, &SDContextParams::photo_maker_path},
+    {"pulid_weights", "PuLID", {"pulid"}, &SDContextParams::pulid_weights_path},
+};
+
+static const ModelSlot* find_model_slot(const std::string& id) {
+    for (const auto& slot : k_model_slots) {
+        if (id == slot.id) {
+            return &slot;
+        }
+    }
+    return nullptr;
+}
+
+static bool matches_model_category(const ModelSlot& slot, const std::string& category) {
+    return std::find(slot.categories.begin(), slot.categories.end(), category) != slot.categories.end();
+}
+
+static void clear_model_paths(SDContextParams& params) {
+    for (const auto& slot : k_model_slots) {
+        (params.*slot.field).clear();
+    }
+}
+
+std::string model_path_key(const fs::path& path) {
+#ifdef _WIN32
+    const auto value = path.generic_wstring();
+    const int length = LCMapStringEx(LOCALE_NAME_INVARIANT, LCMAP_LOWERCASE, value.data(),
+                                     static_cast<int>(value.size()), nullptr, 0, nullptr, nullptr, 0);
+    if (length > 0) {
+        std::wstring folded(length, L'\0');
+        LCMapStringEx(LOCALE_NAME_INVARIANT, LCMAP_LOWERCASE, value.data(),
+                      static_cast<int>(value.size()), &folded[0], length, nullptr, nullptr, 0);
+        return fs::path(folded).generic_u8string();
+    }
+#endif
+    return path.generic_u8string();
+}
+
+bool model_path_inside(const fs::path& root, const fs::path& path) {
+    auto parent = root.begin();
+    auto child  = path.begin();
+    for (; parent != root.end(); ++parent, ++child) {
+        if (child == path.end() || model_path_key(*parent) != model_path_key(*child)) {
+            return false;
+        }
+    }
+    return child != path.end();
+}
+
+ModelRoot make_model_root(const fs::path& path, bool primary) {
+    ModelRoot root;
+    const auto absolute = fs::absolute(path);
+    std::error_code ec;
+    root.path = fs::weakly_canonical(absolute, ec);
+    if (ec)
+        root.path = absolute.lexically_normal();
+    root.primary  = primary;
+    uint64_t hash = 14695981039346656037ULL;
+    for (unsigned char c : model_path_key(root.path)) {
+        hash ^= c;
+        hash *= 1099511628211ULL;
+    }
+    std::ostringstream id;
+    id << '@' << std::hex << std::setfill('0') << std::setw(16) << hash;
+    root.id    = id.str();
+    root.label = root.path.filename().u8string() + " [" + root.id.substr(1, 8) + "]";
+    return root;
+}
+
+std::vector<ModelRoot> get_model_roots(const SDSvrParams& params) {
+    const auto directories = params.model_dirs.empty() ? std::vector<std::string>{"models"} : params.model_dirs;
+    std::vector<ModelRoot> roots;
+    std::set<std::string> seen;
+    for (size_t i = 0; i < directories.size(); ++i) {
+        auto root = make_model_root(fs::u8path(directories[i]), i == 0);
+        if (seen.insert(model_path_key(root.path)).second)
+            roots.push_back(std::move(root));
+    }
+    return roots;
+}
+
+std::vector<fs::path> model_category_directories(const ModelRoot& root, const std::string& category) {
+    std::vector<fs::path> directories;
+    if (!fs::exists(root.path))
+        return directories;
+    for (const auto& entry : fs::directory_iterator(root.path)) {
+        auto name = entry.path().filename().u8string();
+        std::transform(name.begin(), name.end(), name.begin(), [](unsigned char c) { return std::tolower(c); });
+        if (name == category || (category == "checkpoints" && name == "stable-diffusion") ||
+            (category == "loras" && name == "lora")) {
+            directories.push_back(entry.path());
+        }
+    }
+    std::sort(directories.begin(), directories.end(), [&](const fs::path& a, const fs::path& b) {
+        const bool a_standard = a.filename().u8string() == category;
+        const bool b_standard = b.filename().u8string() == category;
+        return a_standard != b_standard ? a_standard : a < b;
+    });
+    return directories;
+}
+
+static bool valid_identifier(const std::string& id) {
+    return !id.empty() && id.size() <= 128 &&
+           std::all_of(id.begin(), id.end(), [](unsigned char c) {
+               return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+                      (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.';
+           });
+}
+
+static void check_keys(const json& object, const std::set<std::string>& keys) {
+    if (!object.is_object()) {
+        throw std::runtime_error("expected an object");
+    }
+    for (auto it = object.begin(); it != object.end(); ++it) {
+        if (!keys.count(it.key())) {
+            throw std::runtime_error("unknown field: " + it.key());
+        }
+    }
+}
+
+static void apply_recipe_defaults(const json& defaults, SDGenerationParams& params) {
+    check_keys(defaults, {"width", "height", "steps", "cfg_scale", "strength", "distilled_guidance", "sampling_method", "scheduler", "custom_sigmas",
+                          "video_frames", "fps", "flow_shift", "moe_boundary", "high_noise_steps", "high_noise_cfg_scale",
+                          "high_noise_flow_shift", "high_noise_sampling_method", "high_noise_scheduler", "img_cfg_scale", "shifted_timestep"});
+    for (const char* key : {"width", "height", "steps", "video_frames", "fps", "high_noise_steps", "shifted_timestep"}) {
+        if (!defaults.contains(key)) {
+            continue;
+        }
+        if (!defaults[key].is_number_integer()) {
+            throw std::runtime_error(std::string(key) + " must be an integer");
+        }
+        const int64_t value     = defaults[key].get<int64_t>();
+        const std::string field = key;
+        const bool dimension    = field == "width" || field == "height";
+        const int64_t maximum   = dimension || field == "video_frames" ? 4096 : field == "fps" ? 240
+                                                                                               : 1000;
+        const int64_t minimum   = dimension ? 64 : field == "shifted_timestep" ? 0
+                                                                               : 1;
+        if ((value < minimum || value > maximum) && !(field == "high_noise_steps" && value == -1)) {
+            throw std::runtime_error(std::string(key) + " is out of range");
+        }
+        if (std::string(key) == "width")
+            params.width = static_cast<int>(value);
+        if (std::string(key) == "height")
+            params.height = static_cast<int>(value);
+        if (std::string(key) == "steps")
+            params.sample_params.sample_steps = static_cast<int>(value);
+        if (field == "shifted_timestep")
+            params.sample_params.shifted_timestep = static_cast<int>(value);
+        if (field == "video_frames")
+            params.video_frames = static_cast<int>(value);
+        if (field == "fps")
+            params.fps = static_cast<int>(value);
+        if (field == "high_noise_steps")
+            params.high_noise_sample_params.sample_steps = static_cast<int>(value);
+    }
+    for (const char* key : {"cfg_scale", "strength", "moe_boundary", "img_cfg_scale", "distilled_guidance", "flow_shift", "high_noise_cfg_scale", "high_noise_flow_shift"}) {
+        if (!defaults.contains(key)) {
+            continue;
+        }
+        if (!defaults[key].is_number()) {
+            throw std::runtime_error(std::string(key) + " must be a number");
+        }
+        const float value = defaults[key].get<float>();
+        if (!std::isfinite(value) || value < 0 || value > ((std::string(key) == "strength" || std::string(key) == "moe_boundary") ? 1.f : 1000.f)) {
+            throw std::runtime_error(std::string(key) + " is out of range");
+        }
+        if (std::string(key) == "strength")
+            params.strength = value;
+        if (std::string(key) == "moe_boundary")
+            params.moe_boundary = value;
+        if (std::string(key) == "cfg_scale")
+            params.sample_params.guidance.txt_cfg = value;
+        if (std::string(key) == "img_cfg_scale")
+            params.sample_params.guidance.img_cfg = value;
+        if (std::string(key) == "distilled_guidance")
+            params.sample_params.guidance.distilled_guidance = value;
+        if (std::string(key) == "flow_shift")
+            params.sample_params.flow_shift = value;
+        if (std::string(key) == "high_noise_cfg_scale")
+            params.high_noise_sample_params.guidance.txt_cfg = value;
+        if (std::string(key) == "high_noise_flow_shift")
+            params.high_noise_sample_params.flow_shift = value;
+    }
+    if (defaults.contains("custom_sigmas")) {
+        const auto& sigmas = defaults.at("custom_sigmas");
+        if (!sigmas.is_array() || sigmas.size() == 1 || sigmas.size() > 1001)
+            throw std::runtime_error("custom_sigmas must be empty or contain 2-1001 values");
+        for (const auto& sigma : sigmas) {
+            if (!sigma.is_number() || !std::isfinite(sigma.get<float>()) || sigma.get<float>() < 0)
+                throw std::runtime_error("custom_sigmas must contain finite nonnegative numbers");
+        }
+        params.custom_sigmas = sigmas.get<std::vector<float>>();
+    }
+    for (const char* key : {"sampling_method", "high_noise_sampling_method"}) {
+        if (!defaults.contains(key))
+            continue;
+        auto& sample    = std::string(key) == "sampling_method" ? params.sample_params : params.high_noise_sample_params;
+        const auto name = defaults.at(key).get<std::string>();
+        bool found      = false;
+        for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) {
+            if (name == sd_sample_method_name(static_cast<sample_method_t>(i))) {
+                sample.sample_method = static_cast<sample_method_t>(i);
+                found                = true;
+                break;
+            }
+        }
+        if (!found)
+            throw std::runtime_error(std::string("unknown ") + key);
+    }
+    for (const char* key : {"scheduler", "high_noise_scheduler"}) {
+        if (!defaults.contains(key))
+            continue;
+        auto& sample    = std::string(key) == "scheduler" ? params.sample_params : params.high_noise_sample_params;
+        const auto name = defaults.at(key).get<std::string>();
+        bool found      = false;
+        for (int i = 0; i < SCHEDULER_COUNT; ++i) {
+            if (name == sd_scheduler_name(static_cast<scheduler_t>(i)) ||
+                (name == "normal" && i == DISCRETE_SCHEDULER)) {
+                sample.scheduler = static_cast<scheduler_t>(i);
+                found            = true;
+                break;
+            }
+        }
+        if (!found)
+            throw std::runtime_error(std::string("unknown ") + key);
+    }
+}
+
+static void apply_recipe_context(const json& settings, SDContextParams& params) {
+    check_keys(settings, {"vae_format", "model_args"});
+    if (settings.contains("vae_format")) {
+        const auto format = settings.at("vae_format").get<std::string>();
+        if (format != "auto" && format != "flux" && format != "sd3" && format != "flux2" && format != "wan")
+            throw std::runtime_error("unknown vae_format");
+        params.vae_format = format;
+    }
+    if (settings.contains("model_args")) {
+        params.model_args = settings.at("model_args").get<std::string>();
+    }
+}
+
+static const std::map<std::string, bool> k_recipe_features = {
+    {"img2img", true},
+    {"mask_image", true},
+    {"lora", true},
+    {"vae_tiling", true},
+    {"cache", true},
+    {"cache_dit", true},
+    {"preview", true},
+    {"flow_shift", false},
+    {"distilled_guidance", false},
+    {"shifted_timestep", false},
+    {"img_cfg", false},
+    {"slg", false},
+    {"clip_skip", false},
+    {"ref_images", false},
+    {"controlnet", false},
+    {"control_image", false},
+    {"ip_adapter", false},
+    {"init_image", false},
+    {"end_image", false},
+    {"control_frames", false},
+    {"vace", false},
+    {"high_noise", false},
+    {"cache_unet", false},
+    {"preview_tae", false},
+};
+
+static void validate_recipe_features(const json& recipe) {
+    for (const char* key : {"features", "disabled_features"}) {
+        if (!recipe.contains(key))
+            continue;
+        const auto& values = recipe.at(key);
+        if (!values.is_array() || values.size() > k_recipe_features.size())
+            throw std::runtime_error(std::string(key) + " must be an array of feature names");
+        std::set<std::string> seen;
+        for (const auto& value : values) {
+            const auto name = value.get<std::string>();
+            if (!k_recipe_features.count(name) || !seen.insert(name).second)
+                throw std::runtime_error("unknown or duplicate feature: " + name);
+        }
+    }
+}
+
+json resolve_recipe_features(const json& recipe) {
+    json features = k_recipe_features;
+    for (const auto& slot : recipe.at("slots")) {
+        const auto id = slot.at("id").get<std::string>();
+        if (id == "ip_adapter")
+            features["ip_adapter"] = true;
+        if (id == "control_net")
+            features["controlnet"] = true;
+        if (id == "high_noise_diffusion_model")
+            features["high_noise"] = true;
+        if (id == "tae" || id == "taesd")
+            features["preview_tae"] = true;
+    }
+    for (const auto& name : recipe.value("features", json::array()))
+        features[name.get<std::string>()] = true;
+    for (const auto& name : recipe.value("disabled_features", json::array()))
+        features[name.get<std::string>()] = false;
+    return features;
+}
+
+json model_generation_features(const ServerRuntime& runtime) {
+    if (!runtime.sd_ctx || runtime.models->active_recipe.is_null()) {
+        json features = k_recipe_features;
+        for (auto& feature : features)
+            feature = false;
+        return features;
+    }
+    json features = resolve_recipe_features(runtime.models->active_recipe);
+    if (runtime.models->active_recipe.value("source", "") == "command_line")
+        return features;
+    // Optional components only become usable once selected and loaded.
+    features["ip_adapter"]  = features.value("ip_adapter", false) && !runtime.ctx_params->ip_adapter_path.empty();
+    features["controlnet"]  = features.value("controlnet", false) && !runtime.ctx_params->control_net_path.empty();
+    features["high_noise"]  = features.value("high_noise", false) && !runtime.ctx_params->high_noise_diffusion_model_path.empty();
+    features["preview_tae"] = features.value("preview_tae", false) && !runtime.ctx_params->taesd_path.empty();
+    if (!runtime.ctx_params->pulid_weights_path.empty())
+        features["slg"] = false;
+    return features;
+}
+
+static void validate_recipe(const json& recipe) {
+    check_keys(recipe, {"schema_version", "id", "name", "description", "model_url", "slots", "required_any", "defaults", "context", "features", "disabled_features"});
+    if (!recipe.at("schema_version").is_number_integer() || recipe.at("schema_version") != 1 ||
+        !valid_identifier(recipe.at("id").get<std::string>()) || recipe.at("name").get<std::string>().empty()) {
+        throw std::runtime_error("invalid schema_version, id or name");
+    }
+    if (recipe.contains("description") && !recipe["description"].is_string()) {
+        throw std::runtime_error("description must be a string");
+    }
+    if (recipe.contains("model_url")) {
+        const auto url           = recipe.at("model_url").get<std::string>();
+        const std::string prefix = "https://huggingface.co/";
+        if (url.size() <= prefix.size() || url.size() > 2048 || url.rfind(prefix, 0) != 0 ||
+            std::any_of(url.begin(), url.end(), [](unsigned char c) { return std::isspace(c) || std::iscntrl(c); })) {
+            throw std::runtime_error("model_url must be a Hugging Face HTTPS URL");
+        }
+    }
+    const auto& slots = recipe.at("slots");
+    if (!slots.is_array() || slots.empty() || slots.size() > 32) {
+        throw std::runtime_error("slots must be a nonempty array of at most 32 entries");
+    }
+    std::set<std::string> ids;
+    bool has_required_model = false;
+    for (const auto& slot : slots) {
+        check_keys(slot, {"id", "label", "category", "required"});
+        const auto id    = slot.at("id").get<std::string>();
+        const auto* spec = find_model_slot(id);
+        if (!spec || !ids.insert(id == "taesd" ? "tae" : id).second ||
+            (slot.contains("category") && !matches_model_category(*spec, slot.at("category").get<std::string>())) ||
+            (slot.contains("label") && slot.at("label").get<std::string>().empty()) || !slot.at("required").is_boolean()) {
+            throw std::runtime_error("invalid or duplicate model slot: " + id);
+        }
+        has_required_model |= (id == "model" || id == "diffusion_model") && slot["required"].get<bool>();
+    }
+    if (!has_required_model) {
+        throw std::runtime_error("a required model or diffusion_model slot is needed");
+    }
+    if (recipe.contains("required_any")) {
+        const auto& groups = recipe.at("required_any");
+        if (!groups.is_array() || groups.size() > 32)
+            throw std::runtime_error("required_any must be an array of at most 32 slot groups");
+        for (const auto& group : groups) {
+            if (!group.is_array() || group.size() < 2 || group.size() > slots.size())
+                throw std::runtime_error("required_any groups need at least two declared slots");
+            std::set<std::string> seen;
+            for (const auto& value : group) {
+                const auto id = value.get<std::string>();
+                if (!seen.insert(id).second || std::none_of(slots.begin(), slots.end(), [&](const json& slot) { return slot.at("id") == id; }))
+                    throw std::runtime_error("unknown or duplicate required_any slot: " + id);
+            }
+        }
+    }
+    validate_recipe_features(recipe);
+    SDContextParams context;
+    apply_recipe_context(recipe.value("context", json::object()), context);
+    SDGenerationParams defaults;
+    apply_recipe_defaults(recipe.value("defaults", json::object()), defaults);
+}
+
+static json normalize_recipe(json recipe) {
+    validate_recipe(recipe);
+    recipe["effective_features"] = resolve_recipe_features(recipe);
+    for (auto& slot : recipe.at("slots")) {
+        const auto& spec   = *find_model_slot(slot.at("id").get<std::string>());
+        slot["label"]      = slot.value("label", std::string(spec.label));
+        slot["category"]   = slot.value("category", spec.categories.front());
+        slot["categories"] = spec.categories;
+    }
+    return recipe;
+}
+
+static bool supported_model_file(const fs::path& path, const std::string& category) {
+    std::string ext = path.extension().u8string();
+    std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });
+    if (category == "tokenizers") {
+        return ext == ".json";
+    }
+    return ext == ".gguf" || ext == ".safetensors" || ext == ".sft" ||
+           ext == ".ckpt" || ext == ".pt" || ext == ".pth" || ext == ".bin";
+}
+
+void refresh_model_catalog(ServerRuntime& runtime) {
+    auto& models = *runtime.models;
+    std::lock_guard<std::mutex> lock(models.catalog_mutex);
+    std::map<std::string, json> recipes;
+    std::map<std::string, ModelFile> files;
+    json errors = json::array();
+    auto report = [&](const std::string& file, const std::string& message) {
+        errors.push_back({{"file", file}, {"message", message}});
+    };
+    for (const char* builtin : k_builtin_recipes) {
+        auto recipe      = normalize_recipe(json::parse(builtin));
+        const auto id    = recipe.at("id").get<std::string>();
+        recipe["source"] = "builtin";
+        recipes[id]      = std::move(recipe);
+    }
+
+    std::set<std::string> seen_files;
+    for (const auto& source : get_model_roots(*runtime.svr_params)) {
+        const auto& root  = source.path;
+        auto display_path = [&](const fs::path& path) {
+            return source.id + "/" + path.lexically_relative(root).generic_u8string();
+        };
+        const fs::path recipe_dir = root / "recipes";
+        std::map<std::string, std::vector<json>> user_recipes;
+        try {
+            if (fs::exists(recipe_dir)) {
+                if (!model_path_inside(root, fs::canonical(recipe_dir)))
+                    throw std::runtime_error("recipes directory escapes model directory");
+                for (const auto& entry : fs::directory_iterator(recipe_dir)) {
+                    const std::string display = display_path(entry.path());
+                    if (entry.path().extension() != ".json")
+                        continue;
+                    try {
+                        if (!entry.is_regular_file() || !model_path_inside(root, fs::canonical(entry.path())))
+                            throw std::runtime_error("recipe must be a file inside the model directory");
+                        if (entry.file_size() > 1024 * 1024)
+                            throw std::runtime_error("recipe exceeds 1 MiB");
+                        std::ifstream input(entry.path(), std::ios::binary);
+                        json recipe;
+                        input >> recipe;
+                        recipe           = normalize_recipe(std::move(recipe));
+                        recipe["source"] = "user";
+                        recipe["file"]   = entry.path().filename().u8string();
+                        recipe["root"]   = source.id;
+                        const auto id    = recipe.at("id").get<std::string>();
+                        user_recipes[id].push_back(std::move(recipe));
+                    } catch (const std::exception& e) {
+                        report(display, e.what());
+                    }
+                }
+            }
+        } catch (const std::exception& e) {
+            report(display_path(recipe_dir), e.what());
+        }
+        for (auto& item : user_recipes) {
+            if (item.second.size() != 1) {
+                for (const auto& recipe : item.second)
+                    report(source.id + "/recipes/" + recipe["file"].get<std::string>(), "duplicate user recipe id: " + item.first);
+                continue;
+            }
+            const auto existing = recipes.find(item.first);
+            if (existing != recipes.end()) {
+                if (existing->second["source"] == "user") {
+                    LOG_WARN("recipe '%s' from %s is shadowed by an earlier model directory", item.first.c_str(), source.label.c_str());
+                    continue;
+                }
+                LOG_INFO("user recipe '%s' from %s overrides the builtin", item.first.c_str(), source.label.c_str());
+            }
+            recipes[item.first] = std::move(item.second.front());
+        }
+
+        std::set<std::string> categories;
+        for (const auto& slot : k_model_slots)
+            categories.insert(slot.categories.begin(), slot.categories.end());
+        for (const auto& category : categories) {
+            try {
+                for (const auto& directory : model_category_directories(source, category)) {
+                    if (!model_path_inside(root, fs::canonical(directory))) {
+                        report(display_path(directory), "category directory escapes model directory");
+                        continue;
+                    }
+                    for (fs::recursive_directory_iterator it(directory), end; it != end; ++it) {
+                        std::error_code path_error;
+                        const fs::path canonical = fs::canonical(it->path(), path_error);
+                        if (path_error || !model_path_inside(root, canonical)) {
+                            it.disable_recursion_pending();
+                            report(display_path(it->path()), path_error ? path_error.message() : "path escapes model directory");
+                            continue;
+                        }
+                        if (!it->is_regular_file() || !supported_model_file(it->path(), category) ||
+                            !seen_files.insert(model_path_key(canonical)).second)
+                            continue;
+                        const auto relative = it->path().lexically_relative(root).generic_u8string();
+                        const auto id       = source.primary ? relative : source.id + "/" + relative;
+                        files[id]           = {category, it->path().lexically_relative(directory).generic_u8string(), relative, source};
+                    }
+                }
+            } catch (const std::exception& e) {
+                report(source.id + "/" + category, e.what());
+            }
+        }
+    }
+    std::map<std::pair<std::string, std::string>, size_t> names;
+    for (const auto& entry : files)
+        ++names[{entry.second.category, entry.second.name}];
+    for (auto& entry : files) {
+        auto& file = entry.second;
+        if (names[{file.category, file.name}] > 1)
+            file.name = (file.root.path / fs::u8path(file.relative_path)).generic_u8string();
+    }
+    models.recipes        = std::move(recipes);
+    models.files          = std::move(files);
+    models.catalog_errors = std::move(errors);
+}
+
+json model_status(ServerRuntime& runtime) {
+    std::lock_guard<std::mutex> lock(runtime.models->status_mutex);
+    auto& models = *runtime.models;
+    json status  = models.status;
+    if (status["state"] == "loading" && !status["load_elapsed_seconds"].is_null()) {
+        status["load_elapsed_seconds"] = std::chrono::duration<double>(std::chrono::steady_clock::now() - models.load_started_at).count();
+    }
+    return status;
+}
+
+void initialize_server_models(ServerRuntime& runtime) {
+    auto& models        = *runtime.models;
+    models.base_context = *runtime.ctx_params;
+    clear_model_paths(models.base_context);
+    models.base_defaults = *runtime.default_gen_params;
+    if (runtime.sd_ctx) {
+        models.status["state"]  = "ready";
+        models.status["origin"] = "command_line";
+        json features           = json::array();
+        for (const auto& feature : k_recipe_features)
+            features.push_back(feature.first);
+        models.active_recipe = {
+            {"id", "@command-line"},
+            {"name", "Command-line model (all features)"},
+            {"source", "command_line"},
+            {"description", "Uses the loaded command-line model and its generation defaults."},
+            {"slots", json::array()},
+            {"defaults", json::object()},
+            {"features", features},
+        };
+        models.active_recipe["effective_features"] = resolve_recipe_features(models.active_recipe);
+        models.status["runtime_recipe"]            = models.active_recipe;
+    }
+    refresh_model_catalog(runtime);
+}
+
+void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe) {
+    check_keys(body, {"recipe", "models"});
+    const auto recipe_id = body.at("recipe").get<std::string>();
+    auto selection       = body.at("models");
+    if (!selection.is_object())
+        throw std::runtime_error("models must be an object");
+    auto& models = *runtime.models;
+    std::lock_guard<std::mutex> lock(models.catalog_mutex);
+    const auto found = models.recipes.find(recipe_id);
+    if (found == models.recipes.end())
+        throw std::runtime_error("unknown recipe");
+    const auto& recipe = found->second;
+    if (loaded_recipe)
+        *loaded_recipe = recipe;
+    context  = models.base_context;
+    defaults = models.base_defaults;
+    apply_recipe_context(recipe.value("context", json::object()), context);
+    apply_recipe_defaults(recipe.value("defaults", json::object()), defaults);
+    std::set<std::string> allowed;
+    for (const auto& slot : recipe.at("slots")) {
+        const auto id = slot.at("id").get<std::string>();
+        allowed.insert(id);
+        if (id == "tae" || id == "taesd") {
+            const std::string alias = id == "tae" ? "taesd" : "tae";
+            allowed.insert(alias);
+            if (selection.contains(alias)) {
+                if (selection.contains(id))
+                    throw std::runtime_error("specify only one of tae and taesd");
+                selection[id] = selection.at(alias);
+            }
+        }
+        const std::string file_id = selection.value(id, std::string());
+        if (file_id.empty()) {
+            if (slot.at("required").get<bool>())
+                throw std::runtime_error("missing required model: " + id);
+            continue;
+        }
+        const auto file = models.files.find(file_id);
+        if (file == models.files.end() || !matches_model_category(*find_model_slot(id), file->second.category)) {
+            throw std::runtime_error("unknown model or incorrect category: " + id);
+        }
+        const auto& root    = file->second.root.path;
+        const fs::path path = fs::canonical(root / fs::u8path(file->second.relative_path));
+        if (!model_path_inside(root, path) || !fs::is_regular_file(path)) {
+            throw std::runtime_error("model file must remain inside the model directory: " + id);
+        }
+        context.*find_model_slot(id)->field = path.u8string();
+    }
+    for (auto it = selection.begin(); it != selection.end(); ++it) {
+        if (!allowed.count(it.key()))
+            throw std::runtime_error("unknown model slot: " + it.key());
+    }
+    for (const auto& group : recipe.value("required_any", json::array())) {
+        if (std::none_of(group.begin(), group.end(), [&](const json& value) {
+                return !selection.value(value.get<std::string>(), std::string()).empty();
+            })) {
+            throw std::runtime_error("select at least one model from: " + group.dump());
+        }
+    }
+    if (!context.resolve_and_validate(IMG_GEN))
+        throw std::runtime_error("invalid model context parameters");
+}
diff --git a/examples/server/model_catalog.h b/examples/server/model_catalog.h
new file mode 100644
index 0000000000..7d49a19241
--- /dev/null
+++ b/examples/server/model_catalog.h
@@ -0,0 +1,56 @@
+#ifndef __SD_EXAMPLES_SERVER_MODEL_CATALOG_H__
+#define __SD_EXAMPLES_SERVER_MODEL_CATALOG_H__
+
+#include <chrono>
+#include <map>
+#include <mutex>
+#include <shared_mutex>
+#include <string>
+
+#include "runtime.h"
+
+struct ModelRoot {
+    std::filesystem::path path;
+    std::string id;
+    std::string label;
+    bool primary = false;
+};
+
+struct ModelFile {
+    std::string category;
+    std::string name;
+    std::string relative_path;
+    ModelRoot root;
+};
+
+std::string model_path_key(const std::filesystem::path& path);
+bool model_path_inside(const std::filesystem::path& root, const std::filesystem::path& path);
+ModelRoot make_model_root(const std::filesystem::path& path, bool primary);
+std::vector<ModelRoot> get_model_roots(const SDSvrParams& params);
+std::vector<std::filesystem::path> model_category_directories(const ModelRoot& root, const std::string& category);
+
+struct ServerModels {
+    std::shared_mutex runtime_mutex;
+    std::mutex catalog_mutex;
+    std::mutex status_mutex;
+    SDCtxPtr context;
+    SDContextParams base_context;
+    SDGenerationParams base_defaults;
+    std::map<std::string, json> recipes;
+    std::map<std::string, ModelFile> files;
+    std::chrono::steady_clock::time_point load_started_at;
+    json active_recipe  = nullptr;
+    json catalog_errors = json::array();
+    json status         = {{"state", "empty"}, {"recipe", nullptr}, {"models", json::object()}, {"error", nullptr}, {"load_elapsed_seconds", nullptr}};
+};
+
+void refresh_model_catalog(ServerRuntime& runtime);
+json model_status(ServerRuntime& runtime);
+void initialize_server_models(ServerRuntime& runtime);
+
+json resolve_recipe_features(const json& recipe);
+json model_generation_features(const ServerRuntime& runtime);
+
+void prepare_model_load(ServerRuntime& runtime, const json& body, SDContextParams& context, SDGenerationParams& defaults, json* loaded_recipe = nullptr);
+
+#endif
diff --git a/examples/server/openai_edit_policy.h b/examples/server/openai_edit_policy.h
new file mode 100644
index 0000000000..c963a887f8
--- /dev/null
+++ b/examples/server/openai_edit_policy.h
@@ -0,0 +1,44 @@
+#ifndef __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
+#define __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
+
+#include "common/resource_owners.hpp"
+
+enum class OpenAIEditInitImageAction {
+    KeepExisting,
+    UseUploaded,
+    Clear,
+};
+
+inline bool openai_edit_uses_uploaded_image_as_init(bool supports_ref_image_conditioning,
+                                                    bool has_ref_images,
+                                                    bool has_mask,
+                                                    bool has_explicit_strength_override) {
+    return !supports_ref_image_conditioning || !has_ref_images || has_mask || has_explicit_strength_override;
+}
+
+inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy(
+    SDImageOwner& init_image,
+    SDImageOwner& uploaded_image,
+    bool uploaded_image_was_first_reference,
+    bool supports_ref_image_conditioning,
+    bool has_ref_images,
+    bool has_mask,
+    bool has_extra_init_image_override,
+    bool has_explicit_strength_override) {
+    if (!uploaded_image_was_first_reference || has_extra_init_image_override) {
+        return OpenAIEditInitImageAction::KeepExisting;
+    }
+
+    if (openai_edit_uses_uploaded_image_as_init(supports_ref_image_conditioning,
+                                                has_ref_images,
+                                                has_mask,
+                                                has_explicit_strength_override)) {
+        init_image = std::move(uploaded_image);
+        return OpenAIEditInitImageAction::UseUploaded;
+    }
+
+    init_image.reset();
+    return OpenAIEditInitImageAction::Clear;
+}
+
+#endif  // __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__
diff --git a/examples/server/recipes/README.md b/examples/server/recipes/README.md
new file mode 100644
index 0000000000..5043f6d1b8
--- /dev/null
+++ b/examples/server/recipes/README.md
@@ -0,0 +1,79 @@
+# Built-in recipe policy
+
+Read these rules before adding or changing JSON files in this directory. A recipe
+represents a supported model-loading and capability configuration, not a saved
+prompt, a style preset, or a collection of sampling settings.
+
+## Required rules
+
+1. **Community editing adapters get a separate recipe.** When the base model
+   does not natively support instruction/reference-image editing and needs a
+   community LoRA to gain it, provide a dedicated editing recipe. Keep editing
+   inputs disabled in the base recipe. Examples: `anima` / `anima-edit` and
+   `krea2-raw` / `krea2-raw-edit`.
+2. **Split different task weights; share native multi-task weights.** If editing
+   and text-to-image use different generation checkpoints, use separate recipes.
+   If the same checkpoint natively supports both, use one recipe. Examples:
+   `boogu-image` / `boogu-image-edit` are separate; `flux2-dev` and
+   `qwen-image-21` each cover both generation and editing. Apply the same rule
+   to video: separate HunyuanVideo 1.5 T2V/I2V and Wan 2.2 A14B T2V/I2V weights,
+   but share one `wan22-ti2v-5b` recipe for its native T2V/I2V checkpoint.
+3. **Capability-extending weights get a separate recipe.** ControlNet,
+   IP-Adapter, PuLID, PhotoMaker, motion modules, and similar adapters that add
+   a distinct conditioning or generation capability belong in dedicated
+   recipes. Make the extension's model slots required when the schema provides
+   them. Do not add these as optional slots on the ordinary base recipe.
+4. **A LoRA alone is not a reason to split.** If adding a LoRA does not change
+   supported functionality, reuse the base recipe. Style, character, quality,
+   acceleration and distillation LoRAs do not get dedicated built-in recipes.
+   Users select the LoRA and change steps, guidance or other generation settings
+   separately. Standalone base/turbo/distilled checkpoints still follow the task,
+   component and layer-count rules; a different filename alone is not a reason
+   for another recipe.
+5. **Keep native components in the model's recipe.** Native vision encoders,
+   projectors, connectors and other components required by the model belong in
+   its existing recipe. Splitting a native component into another file does not
+   make it a capability-extending adapter under rule 3. For example, Qwen vision
+   projectors and LLaDA/LTX connectors do not get separate recipes. A slot may be
+   optional when its weights can be bundled in another selected file.
+6. **Layer-count variants share the family recipe.** Adding, repeating, removing
+   or pruning model layers does not justify a separate recipe when the runtime
+   supports the variant with the same components and capabilities. This also
+   applies to distilled checkpoints whose relevant difference is reduced depth.
+   Use `flux1-dev` for FLUX.1 Lite, `sdxl-checkpoint` for SSD-1B and Segmind Vega,
+   and the corresponding SD 1/2 checkpoint recipes for Tiny SD/Tiny SD2. Only an
+   independent difference in task, required component compatibility or latent
+   format can justify splitting such a variant; document that difference.
+
+## Interpretation and review
+
+- Instruction/reference-image editing is distinct from generic image-to-image
+  denoising. The shared `img2img` capability does not imply native editing.
+- Native components and external capability adapters are distinguished by their
+  role, not by whether their weights are stored in a separate file. LLaDA's
+  SigVQ-containing connectors remain in its shared generation/editing recipe.
+- Do not merge models solely because they descend from the same family. Flex.2
+  has different native inpainting/control conditioning, beyond its layer count
+  ([model card](https://huggingface.co/ostris/Flex.2-preview)). SDXS uses an
+  adjusted AutoEncoderTiny and a one-step model, rather than only a shallower
+  ordinary SD checkpoint ([runtime guide](../../../docs/distilled_sd.md#sdxs-512-dreamshaper)).
+  Their native components stay in their own model recipe.
+- Quantization, file format, prompt wording, seeds, step counts and CFG values
+  are not grounds for another recipe. Differences in required components,
+  latent formats or supported capabilities must be documented.
+- Evaluate what an adapter actually changes. Do not infer new functionality just
+  from a LoRA's name or from the presence of another weight file. In particular,
+  do not create an Omni-Kontext recipe solely because a LoRA is selected.
+- A recipe does not load generation LoRAs automatically. Community-edit recipes
+  must explain that the matching LoRA is required and must be selected separately.
+- Only advertise features implemented by the current runtime. Check the relevant
+  model documentation and code, and match each task's slots and feature flags to
+  its weights. Record any unverified inference paths in the change description.
+- Before submitting, validate all recipe JSON, inspect the embedded recipe API,
+  and check that task-specific inputs appear only on the appropriate recipes.
+  A catalog check is not a substitute for model-generation verification.
+
+These are built-in catalog rules. Users may keep personal presets in
+`<model-dir>/recipes/`; they need not add those presets to the built-in list.
+See [the recipe schema and feature reference](../api.md#user-recipe-format)
+for supported fields.
diff --git a/examples/server/recipes/anima-edit.json b/examples/server/recipes/anima-edit.json
new file mode 100644
index 0000000000..f22c1b6ccc
--- /dev/null
+++ b/examples/server/recipes/anima-edit.json
@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "id": "anima-edit",
+  "name": "Anima Edit (LoRA)",
+  "description": "Requires a matching community editing LoRA. Apply it after loading and supply reference images; uses the cosmos_reference preset.",
+  "model_url": "https://huggingface.co/circlestone-labs/Anima",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 0.6B Base Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "scheduler": "discrete",
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/anima.json b/examples/server/recipes/anima.json
new file mode 100644
index 0000000000..4809bd4bfb
--- /dev/null
+++ b/examples/server/recipes/anima.json
@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "id": "anima",
+  "name": "Anima",
+  "description": "Supports all Anima variants supported by sd.cpp. Use a compatible Qwen Image VAE or TAE; adjust generation settings for distilled variants.",
+  "model_url": "https://huggingface.co/circlestone-labs/Anima",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 0.6B Base Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/animatediff-sd15.json b/examples/server/recipes/animatediff-sd15.json
new file mode 100644
index 0000000000..a7c8338f19
--- /dev/null
+++ b/examples/server/recipes/animatediff-sd15.json
@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "id": "animatediff-sd15",
+  "name": "AnimateDiff SD 1.5",
+  "description": "Use Video Generation with an SD 1.5 checkpoint and compatible motion module.",
+  "model_url": "https://huggingface.co/guoyww/animatediff",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "motion_module",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 8,
+    "fps": 8,
+    "cfg_scale": 7.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "init_image"
+  ],
+  "disabled_features": [
+    "cache_dit",
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/boogu-image-edit.json b/examples/server/recipes/boogu-image-edit.json
new file mode 100644
index 0000000000..ff1aaeefb2
--- /dev/null
+++ b/examples/server/recipes/boogu-image-edit.json
@@ -0,0 +1,50 @@
+{
+  "schema_version": 1,
+  "id": "boogu-image-edit",
+  "name": "Boogu Image Edit",
+  "description": "Supply reference images. The separate vision projector is needed only when it is not bundled in the text encoder.",
+  "model_url": "https://huggingface.co/Boogu/Boogu-Image-0.1-Edit",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 8B Text Encoder"
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "img_cfg_scale": 1,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/boogu-image.json b/examples/server/recipes/boogu-image.json
new file mode 100644
index 0000000000..24c6a42b24
--- /dev/null
+++ b/examples/server/recipes/boogu-image.json
@@ -0,0 +1,41 @@
+{
+  "schema_version": 1,
+  "id": "boogu-image",
+  "name": "Boogu Image Base",
+  "model_url": "https://huggingface.co/Boogu/Boogu-Image-0.1-Base",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 8B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/chroma-radiance.json b/examples/server/recipes/chroma-radiance.json
new file mode 100644
index 0000000000..6a0be8a562
--- /dev/null
+++ b/examples/server/recipes/chroma-radiance.json
@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "id": "chroma-radiance",
+  "name": "Chroma Radiance",
+  "description": "Pixel-space generation without a VAE. Also accepts the x0 and experimental patch-32 variants.",
+  "model_url": "https://huggingface.co/lodestones/Chroma1-Radiance",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 3.5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift",
+    "slg"
+  ],
+  "disabled_features": [
+    "preview",
+    "vae_tiling"
+  ]
+}
diff --git a/examples/server/recipes/chroma.json b/examples/server/recipes/chroma.json
new file mode 100644
index 0000000000..b8d1da0808
--- /dev/null
+++ b/examples/server/recipes/chroma.json
@@ -0,0 +1,41 @@
+{
+  "schema_version": 1,
+  "id": "chroma",
+  "name": "Chroma",
+  "model_url": "https://huggingface.co/lodestones/Chroma1-HD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/cosxl-edit.json b/examples/server/recipes/cosxl-edit.json
new file mode 100644
index 0000000000..49407fccd1
--- /dev/null
+++ b/examples/server/recipes/cosxl-edit.json
@@ -0,0 +1,36 @@
+{
+  "schema_version": 1,
+  "id": "cosxl-edit",
+  "name": "CosXL Edit",
+  "description": "Supply the image to edit as a reference image.",
+  "model_url": "https://huggingface.co/stabilityai/cosxl",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "img_cfg_scale": 1.5,
+    "sampling_method": "euler_a",
+    "scheduler": "karras"
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "img_cfg",
+    "ref_images",
+    "shifted_timestep"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/defaults.md b/examples/server/recipes/defaults.md
new file mode 100644
index 0000000000..2be4dfbf68
--- /dev/null
+++ b/examples/server/recipes/defaults.md
@@ -0,0 +1,124 @@
+# Built-in generation defaults
+
+Reviewed against model-publisher cards and reference implementations on 2026-10-20.
+
+These are starting values, not limits on supported resolutions or generation settings.
+Built-in step defaults above 20 are reduced to 20; the cited sources may recommend more steps.
+Users can still choose a higher step count.
+When several aspect ratios or ranges are published, the selected example or square bucket is noted below.
+A dash means no recipe override: the available primary sources did not establish a fixed value, or the size depends on the input image.
+Ordinary DiT recipes explicitly use Euler; PiD retains its LCM sampling path.
+Guidance-distilled models use `cfg_scale: 1` to disable external CFG in sd.cpp. FLUX embedded guidance is stored separately as `distilled_guidance`.
+For community edit adapters and other optional extensions, these are the base-model settings unless the source specifies otherwise.
+
+| Recipe | CFG | Steps | Width × height | Source / notes |
+| --- | --- | --- | --- | --- |
+| `anima` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/circlestone-labs/Anima) The official ranges are 30–50 steps / CFG 4–5; CFG 4 selects the lower end. Edit uses the same base; tune the chosen editing LoRA separately. |
+| `anima-edit` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/circlestone-labs/Anima) The official ranges are 30–50 steps / CFG 4–5; CFG 4 selects the lower end. Edit uses the same base; tune the chosen editing LoRA separately. |
+| `animatediff-sd15` | 7.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/guoyww/AnimateDiff/main/configs/prompts/1_animate/1_3_animate_ToonYou.yaml)  |
+| `boogu-image` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/Boogu/Boogu-Image-0.1-Base)  |
+| `boogu-image-edit` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/Boogu/Boogu-Image-0.1-Edit)  |
+| `chroma` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/lodestones/Chroma1-HD/raw/main/README.md)  |
+| `chroma-radiance` | 3.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/lodestones/Chroma1-Radiance/raw/main/inference_reference_code.py)  |
+| `cosxl-edit` | — | — | — | [Source](https://huggingface.co/stabilityai/cosxl) No numerical generation defaults in the author’s model card; previous test values were removed. |
+| `ernie-image` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/baidu/ERNIE-Image)  |
+| `ernie-image-turbo` | 1 | 8 | 1024 × 1024 | [Source](https://huggingface.co/baidu/ERNIE-Image-Turbo)  |
+| `flex2` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/ostris/Flex.2-preview)  |
+| `flux1-canny` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev)  |
+| `flux1-depth` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev)  |
+| `flux1-dev` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-dev)  |
+| `flux1-fill` | 1 | 20 | 1232 × 1632 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev)  |
+| `flux1-kontext` | 1 | 20 | — | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/flux/pipeline_flux_kontext.py) Output size normally follows the reference image; no fixed size is imposed by this recipe. |
+| `flux1-pulid` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-dev) FLUX.1-dev defaults; identity conditioning is provided by PuLID. |
+| `flux1-schnell` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.1-schnell)  |
+| `flux2-dev` | 1 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-dev)  |
+| `flux2-klein-4b` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-4B)  |
+| `flux2-klein-9b` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-9B)  |
+| `flux2-klein-base-4b` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B)  |
+| `flux2-klein-base-9b` | 4 | 20 | 1024 × 1024 | [Source](https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B)  |
+| `flux3-klein` | 1 | 16 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/black-forest-labs/flux-3-image-klein/main/README.md)  |
+| `flux3-klein-base` | 4 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/black-forest-labs/flux-3-image-klein/main/src/flux3/pipeline.py)  |
+| `hidream-o1-dev` | 1 | 20 | 2048 × 2048 | [Source](https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev)  |
+| `hunyuan-video-15-i2v` | 6 | 20 | 1280 × 720 | [Source](https://huggingface.co/tencent/HunyuanVideo-1.5) 720p base I2V defaults; CFG/step-distilled and 480p checkpoints use different settings. |
+| `hunyuan-video-15-t2v` | 6 | 20 | 1280 × 720 | [Source](https://huggingface.co/tencent/HunyuanVideo-1.5) 720p base T2V defaults; CFG-distilled / 480p checkpoints need their published settings. |
+| `ideogram4` | 7 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/ideogram-oss/ideogram4/main/docs/inference.md) Constant-guidance pipeline defaults; this recipe does not reproduce the CLI’s per-step guidance schedule. |
+| `instruct-pix2pix` | — | 10 | — | [Source](https://huggingface.co/timbrooks/instruct-pix2pix) Official quick-start example; output size follows the input image. |
+| `iris-3b` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/speridlabs/iris-3b)  |
+| `krea2-raw` | 3.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Raw) Base model defaults; edit requires a matching community LoRA with its own settings. |
+| `krea2-raw-edit` | 3.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/krea/Krea-2-Raw) Base model defaults; edit requires a matching community LoRA with its own settings. |
+| `krea2-turbo` | 1 | 8 | 2048 × 2048 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. |
+| `krea2-turbo-edit` | 1 | 8 | 2048 × 2048 | [Source](https://huggingface.co/krea/Krea-2-Turbo) Official CLI example; edit requires a matching community LoRA. |
+| `lens` | 5 | 20 | 1440 × 1440 | [Source](https://huggingface.co/microsoft/Lens) Official model card: RL model, default base_resolution 1440. |
+| `lens-turbo` | 1 | 4 | 1440 × 1440 | [Source](https://huggingface.co/microsoft/Lens-Turbo) Official model card: 4-step distilled model, default base_resolution 1440. |
+| `lingbot-video-dense` | 3 | 20 | 832 × 480 | [Source](https://huggingface.co/robbyant/lingbot-video-dense-1.3b) Official first-stage inference example; optional refiner is separate. |
+| `lingbot-video-moe` | 3 | 20 | 832 × 480 | [Source](https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b) Official inference example. |
+| `llada-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/inclusionAI/LLaDA-Image)  |
+| `llada-image-turbo` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/inclusionAI/LLaDA-Image-Turbo)  |
+| `longcat-image` | 4 | 20 | 1344 × 768 | [Source](https://huggingface.co/meituan-longcat/LongCat-Image)  |
+| `longcat-image-edit` | 4.5 | 20 | — | [Source](https://huggingface.co/meituan-longcat/LongCat-Image-Edit) Reference-image-dependent output size; no fixed size is imposed. |
+| `ltx23-dev` | 3 | 20 | 768 × 512 | [Source](https://raw.githubusercontent.com/Lightricks/LTX-2/main/packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py) Official single-stage parameters, before optional latent upscaling. |
+| `ltx23-distilled` | 1 | 8 | 768 × 512 | [Source](https://raw.githubusercontent.com/Lightricks/LTX-2/main/packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py) Model card specifies 8 steps / CFG 1; size uses official pipeline’s first stage. |
+| `ltx25-dev` | 3 | 20 | 768 × 512 | [Source](https://raw.githubusercontent.com/Lightricks/LTX-2/main/packages/ltx-pipelines/src/ltx_pipelines/utils/constants.py) Official non-distilled pipeline parameters for model versions >= 2.4, before optional upscaling. |
+| `mage-flow` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow)  |
+| `mage-flow-edit` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow)  |
+| `mage-flow-turbo` | 1 | 4 | 1024 × 1024 | [Source](https://huggingface.co/microsoft/Mage-Flow)  |
+| `ming-image` | 1 | 12 | 2048 × 2048 | [Source](https://huggingface.co/inclusionAI/Ming-Image-0.1-Design)  |
+| `minimax-h3-fl2va` | 1 | — | 1344 × 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. |
+| `minimax-h3-ref2va` | 1 | — | 1344 × 768 | [Source](https://huggingface.co/docs/diffusers/main/en/api/pipelines/minimax_h3) Guidance-distilled; official 768p canvas. No fixed step count is prescribed here; the upstream API counts sigma grid points rather than sd.cpp denoising steps. |
+| `minit2i` | 2.5 | 20 | 512 × 512 | [Source](https://huggingface.co/MiniT2I/MiniT2I/raw/main/pipeline.py) B/16 example and native model size; L/16 uses CFG 6. |
+| `moss-image-base` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base)  |
+| `moss-image-sft` | 3 | 20 | 1024 × 1024 | [Source](https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT)  |
+| `nitrosd-realism` | 1 | 1 | 1024 × 1024 | [Source](https://huggingface.co/ChenDY/NitroFusion)  |
+| `ovis-image` | 5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/ATH-MaaS/Ovis-Image-7B)  |
+| `pid` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
+| `pid-15` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
+| `pid-15-flux2` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
+| `pid-15-qwen-image` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
+| `pid-15-sd3` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
+| `pid-flux2` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
+| `pid-qwen-image` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
+| `pid-sd3` | 1 | 4 | 2048 × 2048 | [Source](https://huggingface.co/nvidia/PiD) Standalone PiD decoding at 2K; guidance is handled by the upstream generator. |
+| `pixart-alpha-1024` | 4.5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py)  |
+| `pixart-alpha-512` | 4.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py)  |
+| `pixart-sigma` | 4.5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py)  |
+| `qwen-image` | 4 | 20 | 1328 × 1328 | [Source](https://huggingface.co/Qwen/Qwen-Image) Official square resolution bucket; true_cfg_scale maps to cfg_scale. |
+| `qwen-image-21` | 1 | 20 | 2048 × 2048 | [Source](https://raw.githubusercontent.com/QwenLM/Qwen-Image-2.1/main/README.md) Original checkpoint defaults; select the matching TAEQI2.1 to replace its VAE. |
+| `qwen-image-21-turbo` | 1 | 8 | 2048 × 2048 | [Model card](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo), [fixed sigmas](https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo/blob/main/model_index.json). Standalone Turbo weights; the fixed 8-step schedule is applied on model load. |
+| `qwen-image-edit` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit) Official editing example; output dimensions follow reference-image aspect ratio. |
+| `qwen-image-edit-2509` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) Official editing example; output dimensions follow reference-image aspect ratio. |
+| `qwen-image-edit-2511` | 4 | 20 | — | [Source](https://huggingface.co/Qwen/Qwen-Image-Edit-2511) Official editing example; output dimensions follow reference-image aspect ratio. |
+| `qwen-image-layered` | 4 | 20 | 640 × 640 | [Source](https://huggingface.co/Qwen/Qwen-Image-Layered) Official recommended 640 resolution bucket; layered RGBA VAE is required. |
+| `sd-turbo` | 1 | 1 | 512 × 512 | [Source](https://huggingface.co/stabilityai/sd-turbo)  |
+| `sd1-checkpoint` | 7.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py)  |
+| `sd1-controlnet` | 7.5 | 20 | 512 × 512 | [Source](https://huggingface.co/lllyasviel/control_v11p_sd15_canny)  |
+| `sd1-inpainting` | 7.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py)  |
+| `sd1-ip-adapter` | 7.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py) SD 1.x base pipeline defaults; adapter strength is configured separately. |
+| `sd2-512-checkpoint` | 7.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py)  |
+| `sd2-768-checkpoint` | 7.5 | 20 | 768 × 768 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py)  |
+| `sd2-inpainting` | 7.5 | 20 | 512 × 512 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py)  |
+| `sd3-medium` | 7 | 20 | 1024 × 1024 | [Source](https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers)  |
+| `sd35-large` | 3.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/stabilityai/stable-diffusion-3.5-large)  |
+| `sd35-medium` | 4.5 | 20 | 1024 × 1024 | [Source](https://huggingface.co/stabilityai/stable-diffusion-3.5-medium)  |
+| `sdxl-checkpoint` | 5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py)  |
+| `sdxl-controlnet` | 5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py) SDXL pipeline defaults; compatible ControlNet weights are required. |
+| `sdxl-inpainting` | 8 | 20 | 1024 × 1024 | [Source](https://huggingface.co/diffusers/stable-diffusion-xl-1.0-inpainting-0.1/raw/main/README.md)  |
+| `sdxl-ip-adapter` | 5 | 20 | 1024 × 1024 | [Source](https://raw.githubusercontent.com/huggingface/diffusers/main/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py) SDXL base pipeline defaults; adapter strength is configured separately. |
+| `sdxl-photomaker` | — | — | 1024 × 1024 | [Source](https://huggingface.co/TencentARC/PhotoMaker) Use settings for the selected compatible SDXL checkpoint; no unique CFG/steps are prescribed by the model card. |
+| `sdxl-turbo` | 1 | 1 | 512 × 512 | [Source](https://huggingface.co/stabilityai/sdxl-turbo)  |
+| `sdxs` | 1 | 1 | 512 × 512 | [Source](https://huggingface.co/IDKiro/sdxs-512-dreamshaper)  |
+| `sefi-image` | 4 | 20 | — | [Source](https://huggingface.co/SeFi-Image/SeFi-Image-5B-Base) Base/RL family defaults; Turbo checkpoints require 4 steps / CFG 1. |
+| `sensenova-u15` | 4 | 20 | 2048 × 2048 | [Source](https://raw.githubusercontent.com/OpenSenseNova/SenseNova-U1/main/examples/README.md)  |
+| `wan21-flf2v` | 5 | 20 | 1280 × 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py)  |
+| `wan21-i2v` | 5 | 20 | 832 × 480 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) 480p base defaults; also accepts compatible SkyReels V2 and single Wan 2.2 I2V experts. |
+| `wan21-t2v` | 5 | 20 | 1280 × 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py) 14B base defaults; single Wan 2.2 experts and acceleration LoRAs need matching settings. |
+| `wan21-vace` | 5 | 20 | 832 × 480 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.1/main/generate.py)  |
+| `wan22-i2v` | 3.5 | 20 | 1280 × 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/wan/configs/wan_i2v_A14B.py)  |
+| `wan22-s2v` | 4.5 | 20 | — | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/wan/configs/wan_s2v_14B.py) The official S2V pipeline derives output dimensions from the input. |
+| `wan22-t2v` | 3 | 20 | 1280 × 720 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/wan/configs/wan_t2v_A14B.py)  |
+| `wan22-ti2v-5b` | 5 | 20 | 1280 × 704 | [Source](https://raw.githubusercontent.com/Wan-Video/Wan2.2/main/wan/configs/wan_ti2v_5B.py)  |
+| `z-image` | 4 | 20 | 720 × 1280 | [Source](https://huggingface.co/Tongyi-MAI/Z-Image)  |
+| `z-image-l2p` | — | — | — | [Source](https://huggingface.co/zhen-nan/L2P) No numerical inference recommendations in the author’s model card; previous test values were removed. |
+| `z-image-turbo` | 1 | 9 | 1024 × 1024 | [Source](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo)  |
+
+TAE alternatives use the model-compatible weights listed by [TAESD](https://github.com/madebyollin/taesd) and [TAEHV](https://github.com/madebyollin/taehv).
+TAE slots are omitted for pixel-space models, PiD, layered RGBA decoding and Mage-VAE paths that the current runtime does not map to a compatible TAE.
diff --git a/examples/server/recipes/ernie-image-turbo.json b/examples/server/recipes/ernie-image-turbo.json
new file mode 100644
index 0000000000..5702f5efcb
--- /dev/null
+++ b/examples/server/recipes/ernie-image-turbo.json
@@ -0,0 +1,41 @@
+{
+  "schema_version": 1,
+  "id": "ernie-image-turbo",
+  "name": "ERNIE Image Turbo",
+  "model_url": "https://huggingface.co/baidu/ERNIE-Image-Turbo",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Ministral 3 3B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 8,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/ernie-image.json b/examples/server/recipes/ernie-image.json
new file mode 100644
index 0000000000..a610000083
--- /dev/null
+++ b/examples/server/recipes/ernie-image.json
@@ -0,0 +1,41 @@
+{
+  "schema_version": 1,
+  "id": "ernie-image",
+  "name": "ERNIE Image",
+  "model_url": "https://huggingface.co/baidu/ERNIE-Image",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Ministral 3 3B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/flex2.json b/examples/server/recipes/flex2.json
new file mode 100644
index 0000000000..bbd900c39a
--- /dev/null
+++ b/examples/server/recipes/flex2.json
@@ -0,0 +1,48 @@
+{
+  "schema_version": 1,
+  "id": "flex2",
+  "name": "Flex.2 Preview",
+  "model_url": "https://huggingface.co/ostris/Flex.2-preview",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "distilled_guidance": 3.5
+  },
+  "features": [
+    "clip_skip",
+    "distilled_guidance",
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux1-canny.json b/examples/server/recipes/flux1-canny.json
new file mode 100644
index 0000000000..de9bb609a4
--- /dev/null
+++ b/examples/server/recipes/flux1-canny.json
@@ -0,0 +1,53 @@
+{
+  "schema_version": 1,
+  "id": "flux1-canny",
+  "name": "FLUX.1 Canny dev",
+  "description": "Select Canny weights and supply a precomputed edge map as Control Image. Edge detection is not performed by this recipe.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-Canny-dev",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "distilled_guidance": 30
+  },
+  "features": [
+    "clip_skip",
+    "distilled_guidance",
+    "flow_shift",
+    "slg",
+    "control_image"
+  ],
+  "disabled_features": [
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/flux1-depth.json b/examples/server/recipes/flux1-depth.json
new file mode 100644
index 0000000000..0efefd2756
--- /dev/null
+++ b/examples/server/recipes/flux1-depth.json
@@ -0,0 +1,54 @@
+{
+  "schema_version": 1,
+  "id": "flux1-depth",
+  "name": "FLUX.1 Depth dev",
+  "description": "Select Depth weights and supply a precomputed depth map as Control Image. Depth estimation is not performed by this recipe.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-Depth-dev",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "distilled_guidance": 10,
+    "scheduler": "flux",
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "clip_skip",
+    "distilled_guidance",
+    "flow_shift",
+    "slg",
+    "control_image"
+  ],
+  "disabled_features": [
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/flux1-dev.json b/examples/server/recipes/flux1-dev.json
new file mode 100644
index 0000000000..93561958a3
--- /dev/null
+++ b/examples/server/recipes/flux1-dev.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "flux1-dev",
+  "name": "FLUX.1 dev",
+  "description": "Supports FLUX.1 dev and compatible layer-count variants such as FLUX.1 Lite. Select the transformer with matching CLIP-L, T5XXL and FLUX.1 VAE.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-dev",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "distilled_guidance": 3.5
+  },
+  "features": [
+    "clip_skip",
+    "distilled_guidance",
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux1-fill.json b/examples/server/recipes/flux1-fill.json
new file mode 100644
index 0000000000..5e6485ad10
--- /dev/null
+++ b/examples/server/recipes/flux1-fill.json
@@ -0,0 +1,52 @@
+{
+  "schema_version": 1,
+  "id": "flux1-fill",
+  "name": "FLUX.1 Fill dev",
+  "description": "Select Fill weights and supply Init Image plus Mask Image for inpainting. Denoising strength defaults to 1 for a full denoising schedule.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-Fill-dev",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "distilled_guidance": 30,
+    "strength": 1.0,
+    "scheduler": "flux",
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 1232,
+    "height": 1632
+  },
+  "features": [
+    "clip_skip",
+    "distilled_guidance",
+    "flow_shift",
+    "slg",
+    "img_cfg"
+  ]
+}
diff --git a/examples/server/recipes/flux1-kontext.json b/examples/server/recipes/flux1-kontext.json
new file mode 100644
index 0000000000..d66e41a263
--- /dev/null
+++ b/examples/server/recipes/flux1-kontext.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "flux1-kontext",
+  "name": "FLUX.1 Kontext dev",
+  "description": "Select FLUX.1 Kontext weights. Supports text-to-image and native reference-image editing; an editing LoRA is not required.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-Kontext-dev",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 20,
+    "distilled_guidance": 2.5
+  },
+  "features": [
+    "clip_skip",
+    "distilled_guidance",
+    "flow_shift",
+    "img_cfg",
+    "ref_images",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux1-pulid.json b/examples/server/recipes/flux1-pulid.json
new file mode 100644
index 0000000000..8203dd1b53
--- /dev/null
+++ b/examples/server/recipes/flux1-pulid.json
@@ -0,0 +1,56 @@
+{
+  "schema_version": 1,
+  "id": "flux1-pulid",
+  "name": "FLUX.1 dev PuLID",
+  "description": "Supply a precomputed PuLID identity embedding with the generation request.",
+  "model_url": "https://huggingface.co/guozinan/PuLID",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "pulid_weights",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "distilled_guidance": 3.5
+  },
+  "features": [
+    "clip_skip",
+    "distilled_guidance",
+    "flow_shift",
+    "slg"
+  ],
+  "disabled_features": [
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux1-schnell.json b/examples/server/recipes/flux1-schnell.json
new file mode 100644
index 0000000000..106fef5506
--- /dev/null
+++ b/examples/server/recipes/flux1-schnell.json
@@ -0,0 +1,46 @@
+{
+  "schema_version": 1,
+  "id": "flux1-schnell",
+  "name": "FLUX.1 schnell",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.1-schnell",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "clip_skip",
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux2-dev.json b/examples/server/recipes/flux2-dev.json
new file mode 100644
index 0000000000..4d27ea16b6
--- /dev/null
+++ b/examples/server/recipes/flux2-dev.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "flux2-dev",
+  "name": "FLUX.2 dev",
+  "description": "Supports text-to-image and reference-image editing.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-dev",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Mistral Small 3.2 24B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "distilled_guidance": 4,
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "distilled_guidance",
+    "flow_shift",
+    "img_cfg",
+    "ref_images",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux2-klein-4b.json b/examples/server/recipes/flux2-klein-4b.json
new file mode 100644
index 0000000000..f09360cfbc
--- /dev/null
+++ b/examples/server/recipes/flux2-klein-4b.json
@@ -0,0 +1,45 @@
+{
+  "schema_version": 1,
+  "id": "flux2-klein-4b",
+  "name": "FLUX.2 Klein 4B",
+  "description": "Supports reference-image editing. The FLUX.2 small decoder is an alternative VAE.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-4B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux2-klein-9b.json b/examples/server/recipes/flux2-klein-9b.json
new file mode 100644
index 0000000000..ad10d6e516
--- /dev/null
+++ b/examples/server/recipes/flux2-klein-9b.json
@@ -0,0 +1,45 @@
+{
+  "schema_version": 1,
+  "id": "flux2-klein-9b",
+  "name": "FLUX.2 Klein 9B",
+  "description": "Supports text-to-image and reference-image editing.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-9B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 8B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux2-klein-base-4b.json b/examples/server/recipes/flux2-klein-base-4b.json
new file mode 100644
index 0000000000..56c69f9801
--- /dev/null
+++ b/examples/server/recipes/flux2-klein-base-4b.json
@@ -0,0 +1,45 @@
+{
+  "schema_version": 1,
+  "id": "flux2-klein-base-4b",
+  "name": "FLUX.2 Klein Base 4B",
+  "description": "Supports text-to-image and reference-image editing.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-4B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux2-klein-base-9b.json b/examples/server/recipes/flux2-klein-base-9b.json
new file mode 100644
index 0000000000..e9bd53c6cf
--- /dev/null
+++ b/examples/server/recipes/flux2-klein-base-9b.json
@@ -0,0 +1,45 @@
+{
+  "schema_version": 1,
+  "id": "flux2-klein-base-9b",
+  "name": "FLUX.2 Klein Base 9B",
+  "description": "Use the 9B base transformer with Qwen 3 8B. Supports reference-image editing.",
+  "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-klein-base-9B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 8B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/flux3-klein-base.json b/examples/server/recipes/flux3-klein-base.json
new file mode 100644
index 0000000000..29f6f68196
--- /dev/null
+++ b/examples/server/recipes/flux3-klein-base.json
@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "id": "flux3-klein-base",
+  "name": "FLUX.3 Image Klein Base",
+  "model_url": "https://huggingface.co/black-forest-labs/flux-3-image-klein-base",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/flux3-klein.json b/examples/server/recipes/flux3-klein.json
new file mode 100644
index 0000000000..164bce12ad
--- /dev/null
+++ b/examples/server/recipes/flux3-klein.json
@@ -0,0 +1,43 @@
+{
+  "schema_version": 1,
+  "id": "flux3-klein",
+  "name": "FLUX.3 Image Klein",
+  "description": "Supports text-to-image and reference-image editing.",
+  "model_url": "https://huggingface.co/black-forest-labs/flux-3-image-klein",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 16,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/hidream-o1-dev.json b/examples/server/recipes/hidream-o1-dev.json
new file mode 100644
index 0000000000..8e95f5ceac
--- /dev/null
+++ b/examples/server/recipes/hidream-o1-dev.json
@@ -0,0 +1,24 @@
+{
+  "schema_version": 1,
+  "id": "hidream-o1-dev",
+  "name": "HiDream O1 Image dev",
+  "description": "Bundled checkpoint; also supports reference-image editing.",
+  "model_url": "https://huggingface.co/HiDream-ai/HiDream-O1-Image-Dev",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 2048,
+    "height": 2048,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/hunyuan-video-15-i2v.json b/examples/server/recipes/hunyuan-video-15-i2v.json
new file mode 100644
index 0000000000..a4d672f7b1
--- /dev/null
+++ b/examples/server/recipes/hunyuan-video-15-i2v.json
@@ -0,0 +1,56 @@
+{
+  "schema_version": 1,
+  "id": "hunyuan-video-15-i2v",
+  "name": "HunyuanVideo 1.5 I2V",
+  "description": "Select HunyuanVideo 1.5 I2V weights and supply a start image in Video Generation.",
+  "model_url": "https://huggingface.co/tencent/HunyuanVideo-1.5",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "t5xxl",
+      "required": true,
+      "label": "ByT5 Small GlyphXL"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "HunyuanVideo 1.5 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false,
+      "label": "TAEHV 1.5"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "cfg_scale": 6,
+    "steps": 20,
+    "width": 1280,
+    "height": 720,
+    "flow_shift": 7
+  },
+  "features": [
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/hunyuan-video-15-t2v.json b/examples/server/recipes/hunyuan-video-15-t2v.json
new file mode 100644
index 0000000000..d477b8063d
--- /dev/null
+++ b/examples/server/recipes/hunyuan-video-15-t2v.json
@@ -0,0 +1,55 @@
+{
+  "schema_version": 1,
+  "id": "hunyuan-video-15-t2v",
+  "name": "HunyuanVideo 1.5 T2V",
+  "description": "Select HunyuanVideo 1.5 T2V weights and use Video Generation. Use the separate I2V recipe for I2V weights.",
+  "model_url": "https://huggingface.co/tencent/HunyuanVideo-1.5",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "t5xxl",
+      "required": true,
+      "label": "ByT5 Small GlyphXL"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "HunyuanVideo 1.5 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false,
+      "label": "TAEHV 1.5"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "cfg_scale": 6,
+    "steps": 20,
+    "width": 1280,
+    "height": 720,
+    "flow_shift": 9
+  },
+  "features": [
+    "flow_shift"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/ideogram4.json b/examples/server/recipes/ideogram4.json
new file mode 100644
index 0000000000..2d5230019b
--- /dev/null
+++ b/examples/server/recipes/ideogram4.json
@@ -0,0 +1,46 @@
+{
+  "schema_version": 1,
+  "id": "ideogram4",
+  "name": "Ideogram 4",
+  "description": "Select the matching unconditional transformer for classifier-free guidance.",
+  "model_url": "https://huggingface.co/ideogram-ai/ideogram-4-fp8",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "uncond_diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 8B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 7,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/instruct-pix2pix.json b/examples/server/recipes/instruct-pix2pix.json
new file mode 100644
index 0000000000..e819509108
--- /dev/null
+++ b/examples/server/recipes/instruct-pix2pix.json
@@ -0,0 +1,36 @@
+{
+  "schema_version": 1,
+  "id": "instruct-pix2pix",
+  "name": "Instruct-Pix2Pix",
+  "description": "Supply the image to edit as a reference image.",
+  "model_url": "https://huggingface.co/timbrooks/instruct-pix2pix",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "img_cfg_scale": 1,
+    "sampling_method": "euler_a",
+    "scheduler": "karras",
+    "steps": 10
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "img_cfg",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/iris-3b.json b/examples/server/recipes/iris-3b.json
new file mode 100644
index 0000000000..c1b5d84fcd
--- /dev/null
+++ b/examples/server/recipes/iris-3b.json
@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "id": "iris-3b",
+  "name": "Iris 3B",
+  "description": "Pixel-space model; no VAE is required.",
+  "model_url": "https://huggingface.co/speridlabs/iris-3b",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ],
+  "disabled_features": [
+    "preview",
+    "vae_tiling"
+  ]
+}
diff --git a/examples/server/recipes/krea2-raw-edit.json b/examples/server/recipes/krea2-raw-edit.json
new file mode 100644
index 0000000000..8d3eb9c7b8
--- /dev/null
+++ b/examples/server/recipes/krea2-raw-edit.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "krea2-raw-edit",
+  "name": "Krea 2 Raw Edit (LoRA)",
+  "description": "Requires a matching Ostris-compatible community editing LoRA. Apply it after loading and supply reference images; uses the krea2_ostris_edit preset. Select a separate vision projector when it is not bundled in the text encoder.",
+  "model_url": "https://huggingface.co/krea/Krea-2-Raw",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "scheduler": "discrete",
+    "sampling_method": "euler",
+    "cfg_scale": 3.5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/krea2-raw.json b/examples/server/recipes/krea2-raw.json
new file mode 100644
index 0000000000..5878291ab5
--- /dev/null
+++ b/examples/server/recipes/krea2-raw.json
@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "id": "krea2-raw",
+  "name": "Krea 2 Raw",
+  "description": "For reference-image editing with a community LoRA, use the corresponding Krea 2 Edit recipe.",
+  "model_url": "https://huggingface.co/krea/Krea-2-Raw",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 3.5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/krea2-turbo-edit.json b/examples/server/recipes/krea2-turbo-edit.json
new file mode 100644
index 0000000000..fa248c083b
--- /dev/null
+++ b/examples/server/recipes/krea2-turbo-edit.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "krea2-turbo-edit",
+  "name": "Krea 2 Turbo Edit (LoRA)",
+  "description": "Requires a matching Ostris-compatible community editing LoRA. Apply it after loading and supply reference images; uses the krea2_ostris_edit preset. Select a separate vision projector when it is not bundled in the text encoder.",
+  "model_url": "https://huggingface.co/krea/Krea-2-Turbo",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "scheduler": "discrete",
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 8,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/krea2-turbo.json b/examples/server/recipes/krea2-turbo.json
new file mode 100644
index 0000000000..23dad12fd5
--- /dev/null
+++ b/examples/server/recipes/krea2-turbo.json
@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "id": "krea2-turbo",
+  "name": "Krea 2 Turbo",
+  "description": "For reference-image editing with a community LoRA, use the corresponding Krea 2 Edit recipe.",
+  "model_url": "https://huggingface.co/krea/Krea-2-Turbo",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 8,
+    "width": 2048,
+    "height": 2048,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/lens-turbo.json b/examples/server/recipes/lens-turbo.json
new file mode 100644
index 0000000000..e9282eafc4
--- /dev/null
+++ b/examples/server/recipes/lens-turbo.json
@@ -0,0 +1,46 @@
+{
+  "schema_version": 1,
+  "id": "lens-turbo",
+  "name": "LENS Turbo",
+  "model_url": "https://huggingface.co/microsoft/Lens-Turbo",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "GPT-OSS 20B Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "GPT-OSS tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 1440,
+    "height": 1440,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/lens.json b/examples/server/recipes/lens.json
new file mode 100644
index 0000000000..0a89542293
--- /dev/null
+++ b/examples/server/recipes/lens.json
@@ -0,0 +1,46 @@
+{
+  "schema_version": 1,
+  "id": "lens",
+  "name": "LENS",
+  "model_url": "https://huggingface.co/microsoft/Lens",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "GPT-OSS 20B Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "GPT-OSS tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1440,
+    "height": 1440,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/lingbot-video-dense.json b/examples/server/recipes/lingbot-video-dense.json
new file mode 100644
index 0000000000..5f313a0cd9
--- /dev/null
+++ b/examples/server/recipes/lingbot-video-dense.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "lingbot-video-dense",
+  "name": "LingBot Video Dense 1.3B",
+  "description": "Use Video Generation.",
+  "model_url": "https://huggingface.co/robbyant/lingbot-video-dense-1.3b",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Wan 2.1 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "video_frames": 33,
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 832,
+    "height": 480,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/lingbot-video-moe.json b/examples/server/recipes/lingbot-video-moe.json
new file mode 100644
index 0000000000..cdbce61ee9
--- /dev/null
+++ b/examples/server/recipes/lingbot-video-moe.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "lingbot-video-moe",
+  "name": "LingBot Video MoE 30B A3B",
+  "description": "Use Video Generation.",
+  "model_url": "https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Wan 2.1 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "video_frames": 33,
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 832,
+    "height": 480,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/llada-image-turbo.json b/examples/server/recipes/llada-image-turbo.json
new file mode 100644
index 0000000000..0589bb3846
--- /dev/null
+++ b/examples/server/recipes/llada-image-turbo.json
@@ -0,0 +1,52 @@
+{
+  "schema_version": 1,
+  "id": "llada-image-turbo",
+  "name": "LLaDA Image Turbo",
+  "description": "Use matching turbo text encoder and connectors. Editing needs connectors that include the SigVQ encoder.",
+  "model_url": "https://huggingface.co/inclusionAI/LLaDA-Image-Turbo",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "LLaDA-Image Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "LLaDA tokenizer.json"
+    },
+    {
+      "id": "embeddings_connectors",
+      "required": true,
+      "label": "LLaDA Turbo Connectors"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/llada-image.json b/examples/server/recipes/llada-image.json
new file mode 100644
index 0000000000..27887c849e
--- /dev/null
+++ b/examples/server/recipes/llada-image.json
@@ -0,0 +1,53 @@
+{
+  "schema_version": 1,
+  "id": "llada-image",
+  "name": "LLaDA Image (base)",
+  "description": "Use matching base transformer, text encoder and connectors; do not mix them with Turbo components. Editing needs connectors with SigVQ and 1024x1024 output.",
+  "model_url": "https://huggingface.co/inclusionAI/LLaDA-Image",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "LLaDA-Image Base Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "LLaDA tokenizer.json"
+    },
+    {
+      "id": "embeddings_connectors",
+      "required": true,
+      "label": "LLaDA Base Connectors"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "scheduler": "llada_image",
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/longcat-image-edit.json b/examples/server/recipes/longcat-image-edit.json
new file mode 100644
index 0000000000..bc15ea2af5
--- /dev/null
+++ b/examples/server/recipes/longcat-image-edit.json
@@ -0,0 +1,43 @@
+{
+  "schema_version": 1,
+  "id": "longcat-image-edit",
+  "name": "LongCat Image Edit",
+  "description": "Supply reference images for editing.",
+  "model_url": "https://huggingface.co/meituan-longcat/LongCat-Image-Edit",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 4.5,
+    "steps": 20,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/longcat-image.json b/examples/server/recipes/longcat-image.json
new file mode 100644
index 0000000000..36e23e85e9
--- /dev/null
+++ b/examples/server/recipes/longcat-image.json
@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "id": "longcat-image",
+  "name": "LongCat Image",
+  "model_url": "https://huggingface.co/meituan-longcat/LongCat-Image",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1344,
+    "height": 768,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/ltx23-dev.json b/examples/server/recipes/ltx23-dev.json
new file mode 100644
index 0000000000..83bb934e86
--- /dev/null
+++ b/examples/server/recipes/ltx23-dev.json
@@ -0,0 +1,60 @@
+{
+  "schema_version": 1,
+  "id": "ltx23-dev",
+  "name": "LTX 2.3 dev",
+  "description": "Supports text, start-image and start/end-image video. Audio VAE is optional for silent video. Configure latent upscaling separately in generation settings. Optional distillation LoRAs use this recipe; select the LoRA and adjust the sampling settings separately.",
+  "model_url": "https://huggingface.co/Lightricks/LTX-2.3",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 3 12B Text Encoder"
+    },
+    {
+      "id": "embeddings_connectors",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "LTX 2.3 Video VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    },
+    {
+      "id": "audio_vae",
+      "required": false,
+      "label": "LTX 2.3 Audio VAE"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "fps": 24,
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 768,
+    "height": 512
+  },
+  "features": [
+    "end_image",
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/ltx23-distilled.json b/examples/server/recipes/ltx23-distilled.json
new file mode 100644
index 0000000000..a2a6ac33c2
--- /dev/null
+++ b/examples/server/recipes/ltx23-distilled.json
@@ -0,0 +1,60 @@
+{
+  "schema_version": 1,
+  "id": "ltx23-distilled",
+  "name": "LTX 2.3 Distilled 1.1",
+  "description": "Supports text, start-image and start/end-image video. Audio VAE is optional for silent video. Configure latent upscaling separately in generation settings.",
+  "model_url": "https://huggingface.co/Lightricks/LTX-2.3",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 3 12B Text Encoder"
+    },
+    {
+      "id": "embeddings_connectors",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "LTX 2.3 Video VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    },
+    {
+      "id": "audio_vae",
+      "required": false,
+      "label": "LTX 2.3 Audio VAE"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 121,
+    "fps": 24,
+    "cfg_scale": 1,
+    "steps": 8,
+    "width": 768,
+    "height": 512
+  },
+  "features": [
+    "end_image",
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/ltx25-dev.json b/examples/server/recipes/ltx25-dev.json
new file mode 100644
index 0000000000..47efbc76fd
--- /dev/null
+++ b/examples/server/recipes/ltx25-dev.json
@@ -0,0 +1,56 @@
+{
+  "schema_version": 1,
+  "id": "ltx25-dev",
+  "name": "LTX 2.5 dev",
+  "description": "Use the conv video VAE and LTX-specific Gemma 4 with bundled projection. Audio VAE is optional for silent video.",
+  "model_url": "https://huggingface.co/Lightricks/LTX-2.5",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "LTX-2.5 Gemma 4 12B with projection"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "LTX 2.5 Conv Video VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    },
+    {
+      "id": "audio_vae",
+      "required": false,
+      "label": "LTX 2.5 Audio VAE"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "fps": 24,
+    "video_frames": 33,
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 768,
+    "height": 512
+  },
+  "features": [
+    "end_image",
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/mage-flow-edit.json b/examples/server/recipes/mage-flow-edit.json
new file mode 100644
index 0000000000..82d66f6bd2
--- /dev/null
+++ b/examples/server/recipes/mage-flow-edit.json
@@ -0,0 +1,39 @@
+{
+  "schema_version": 1,
+  "id": "mage-flow-edit",
+  "name": "Mage Flow Edit",
+  "description": "Supply reference images. Select a separate vision projector when it is not bundled in the text encoder.",
+  "model_url": "https://huggingface.co/microsoft/Mage-Flow-Edit",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": true
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/mage-flow-turbo.json b/examples/server/recipes/mage-flow-turbo.json
new file mode 100644
index 0000000000..5cf9bb25a7
--- /dev/null
+++ b/examples/server/recipes/mage-flow-turbo.json
@@ -0,0 +1,31 @@
+{
+  "schema_version": 1,
+  "id": "mage-flow-turbo",
+  "name": "Mage Flow Turbo",
+  "model_url": "https://huggingface.co/microsoft/Mage-Flow-Turbo",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/mage-flow.json b/examples/server/recipes/mage-flow.json
new file mode 100644
index 0000000000..4cbd15934c
--- /dev/null
+++ b/examples/server/recipes/mage-flow.json
@@ -0,0 +1,31 @@
+{
+  "schema_version": 1,
+  "id": "mage-flow",
+  "name": "Mage Flow",
+  "model_url": "https://huggingface.co/microsoft/Mage-Flow",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/ming-image.json b/examples/server/recipes/ming-image.json
new file mode 100644
index 0000000000..6c368e8e1a
--- /dev/null
+++ b/examples/server/recipes/ming-image.json
@@ -0,0 +1,46 @@
+{
+  "schema_version": 1,
+  "id": "ming-image",
+  "name": "Ming Image 0.1",
+  "model_url": "https://huggingface.co/inclusionAI/Ming-Image-0.1-Design",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Ling Mini 2.0 Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Ling tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 12,
+    "width": 2048,
+    "height": 2048,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/minimax-h3-fl2va.json b/examples/server/recipes/minimax-h3-fl2va.json
new file mode 100644
index 0000000000..3971923917
--- /dev/null
+++ b/examples/server/recipes/minimax-h3-fl2va.json
@@ -0,0 +1,55 @@
+{
+  "schema_version": 1,
+  "id": "minimax-h3-fl2va",
+  "name": "MiniMax H3 FL2VA",
+  "description": "Generate video and audio, optionally with start/end images. Supports full and pruned FL2VA weights. Audio VAE is optional for silent video.",
+  "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "MiniMax H3 Video VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    },
+    {
+      "id": "audio_vae",
+      "required": false,
+      "label": "MiniMax H3 Audio VAE"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "video_frames": 56,
+    "fps": 24,
+    "cfg_scale": 1,
+    "width": 1344,
+    "height": 768,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "end_image",
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/minimax-h3-ref2va.json b/examples/server/recipes/minimax-h3-ref2va.json
new file mode 100644
index 0000000000..085f5d5fed
--- /dev/null
+++ b/examples/server/recipes/minimax-h3-ref2va.json
@@ -0,0 +1,54 @@
+{
+  "schema_version": 1,
+  "id": "minimax-h3-ref2va",
+  "name": "MiniMax H3 Ref2VA",
+  "description": "Supply image, video or audio references with compatible Ref2VA weights. Audio VAE is optional for silent video.",
+  "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-H3",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "MiniMax-H3 Qwen 3 VL 32B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "MiniMax H3 Video VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    },
+    {
+      "id": "audio_vae",
+      "required": false,
+      "label": "MiniMax H3 Audio VAE"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "video_frames": 56,
+    "fps": 24,
+    "cfg_scale": 1,
+    "width": 1344,
+    "height": 768,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/minit2i.json b/examples/server/recipes/minit2i.json
new file mode 100644
index 0000000000..2c7b94eba6
--- /dev/null
+++ b/examples/server/recipes/minit2i.json
@@ -0,0 +1,29 @@
+{
+  "schema_version": 1,
+  "id": "minit2i",
+  "name": "MiniT2I",
+  "description": "Pixel-space model; no VAE is required.",
+  "model_url": "https://huggingface.co/MiniT2I/MiniT2I",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true,
+      "label": "FLAN-T5 Large"
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 2.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512,
+    "sampling_method": "euler"
+  },
+  "disabled_features": [
+    "preview",
+    "vae_tiling"
+  ]
+}
diff --git a/examples/server/recipes/moss-image-base.json b/examples/server/recipes/moss-image-base.json
new file mode 100644
index 0000000000..bc2cc7d9b1
--- /dev/null
+++ b/examples/server/recipes/moss-image-base.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "moss-image-base",
+  "name": "MOSS Image Base",
+  "description": "Select the MOSS-Image-Base transformer with its matching text encoder and FLUX.2 VAE or TAE.",
+  "model_url": "https://huggingface.co/OpenMOSS-Team/MOSS-Image-Base",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3.5 9B Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Qwen 3.5 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "flow_shift": 5,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/moss-image-sft.json b/examples/server/recipes/moss-image-sft.json
new file mode 100644
index 0000000000..e0239b0bcd
--- /dev/null
+++ b/examples/server/recipes/moss-image-sft.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "moss-image-sft",
+  "name": "MOSS Image SFT",
+  "description": "Select the MOSS-Image-SFT transformer with its matching text encoder and FLUX.2 VAE or TAE.",
+  "model_url": "https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3.5 9B Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Qwen 3.5 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "flow_shift": 5,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/nitrosd-realism.json b/examples/server/recipes/nitrosd-realism.json
new file mode 100644
index 0000000000..75f2d43ff8
--- /dev/null
+++ b/examples/server/recipes/nitrosd-realism.json
@@ -0,0 +1,36 @@
+{
+  "schema_version": 1,
+  "id": "nitrosd-realism",
+  "name": "NitroSD Realism",
+  "model_url": "https://huggingface.co/ChenDY/NitroFusion",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "scheduler": "sgm_uniform",
+    "shifted_timestep": 250,
+    "cfg_scale": 1,
+    "steps": 1,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "shifted_timestep"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/ovis-image.json b/examples/server/recipes/ovis-image.json
new file mode 100644
index 0000000000..21e9d9fcea
--- /dev/null
+++ b/examples/server/recipes/ovis-image.json
@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "id": "ovis-image",
+  "name": "Ovis Image",
+  "model_url": "https://huggingface.co/ATH-MaaS/Ovis-Image-7B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Ovis 2.5 Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/pid-15-flux2.json b/examples/server/recipes/pid-15-flux2.json
new file mode 100644
index 0000000000..4fb3db3a72
--- /dev/null
+++ b/examples/server/recipes/pid-15-flux2.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "pid-15-flux2",
+  "name": "PiD 1.5 (Flux.2 VAE)",
+  "description": "Supply a reference image; the first image is used as the source latent. Select a matching Flux.2 PiD checkpoint and VAE.",
+  "model_url": "https://huggingface.co/nvidia/PiD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 2 2B ELM Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Gemma 2 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "context": {
+    "vae_format": "flux2"
+  },
+  "defaults": {
+    "sampling_method": "lcm",
+    "scheduler": "lcm",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/pid-15-qwen-image.json b/examples/server/recipes/pid-15-qwen-image.json
new file mode 100644
index 0000000000..9db9e7422b
--- /dev/null
+++ b/examples/server/recipes/pid-15-qwen-image.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "pid-15-qwen-image",
+  "name": "PiD 1.5 (Qwen Image VAE)",
+  "description": "Supply a reference image; the first image is used as the source latent. Select a matching Qwen Image PiD checkpoint and VAE. Use the Qwen Image 2D VAE.",
+  "model_url": "https://huggingface.co/nvidia/PiD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 2 2B ELM Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Gemma 2 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "context": {
+    "vae_format": "wan"
+  },
+  "defaults": {
+    "sampling_method": "lcm",
+    "scheduler": "lcm",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/pid-15-sd3.json b/examples/server/recipes/pid-15-sd3.json
new file mode 100644
index 0000000000..3980b86710
--- /dev/null
+++ b/examples/server/recipes/pid-15-sd3.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "pid-15-sd3",
+  "name": "PiD 1.5 (SD3 VAE)",
+  "description": "Supply a reference image; the first image is used as the source latent. Select a matching SD3 PiD checkpoint and VAE.",
+  "model_url": "https://huggingface.co/nvidia/PiD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 2 2B ELM Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Gemma 2 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "context": {
+    "vae_format": "sd3"
+  },
+  "defaults": {
+    "sampling_method": "lcm",
+    "scheduler": "lcm",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/pid-15.json b/examples/server/recipes/pid-15.json
new file mode 100644
index 0000000000..f7913bee8e
--- /dev/null
+++ b/examples/server/recipes/pid-15.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "pid-15",
+  "name": "PiD 1.5 (Flux / Z-Image VAE)",
+  "description": "Supply a reference image; the first image is used as the source latent. Select a Flux/Z-Image PiD checkpoint and matching Flux VAE.",
+  "model_url": "https://huggingface.co/nvidia/PiD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 2 2B ELM Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Gemma 2 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "context": {
+    "vae_format": "flux"
+  },
+  "defaults": {
+    "sampling_method": "lcm",
+    "scheduler": "lcm",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/pid-flux2.json b/examples/server/recipes/pid-flux2.json
new file mode 100644
index 0000000000..9c802dc153
--- /dev/null
+++ b/examples/server/recipes/pid-flux2.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "pid-flux2",
+  "name": "PiD (Flux.2 VAE)",
+  "description": "Supply a reference image; the first image is used as the source latent. Select a matching Flux.2 PiD checkpoint and VAE.",
+  "model_url": "https://huggingface.co/nvidia/PiD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 2 2B ELM Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Gemma 2 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "context": {
+    "vae_format": "flux2"
+  },
+  "defaults": {
+    "sampling_method": "lcm",
+    "scheduler": "lcm",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/pid-qwen-image.json b/examples/server/recipes/pid-qwen-image.json
new file mode 100644
index 0000000000..4503b1cd69
--- /dev/null
+++ b/examples/server/recipes/pid-qwen-image.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "pid-qwen-image",
+  "name": "PiD (Qwen Image VAE)",
+  "description": "Supply a reference image; the first image is used as the source latent. Select a matching Qwen Image PiD checkpoint and VAE. Use the Qwen Image 2D VAE.",
+  "model_url": "https://huggingface.co/nvidia/PiD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 2 2B ELM Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Gemma 2 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "context": {
+    "vae_format": "wan"
+  },
+  "defaults": {
+    "sampling_method": "lcm",
+    "scheduler": "lcm",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/pid-sd3.json b/examples/server/recipes/pid-sd3.json
new file mode 100644
index 0000000000..6dad2ca09e
--- /dev/null
+++ b/examples/server/recipes/pid-sd3.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "pid-sd3",
+  "name": "PiD (SD3 VAE)",
+  "description": "Supply a reference image; the first image is used as the source latent. Select a matching SD3 PiD checkpoint and VAE.",
+  "model_url": "https://huggingface.co/nvidia/PiD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 2 2B ELM Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Gemma 2 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "context": {
+    "vae_format": "sd3"
+  },
+  "defaults": {
+    "sampling_method": "lcm",
+    "scheduler": "lcm",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/pid.json b/examples/server/recipes/pid.json
new file mode 100644
index 0000000000..2479449659
--- /dev/null
+++ b/examples/server/recipes/pid.json
@@ -0,0 +1,47 @@
+{
+  "schema_version": 1,
+  "id": "pid",
+  "name": "PiD (Flux / Z-Image VAE)",
+  "description": "Supply a reference image; the first image is used as the source latent. Select a Flux/Z-Image PiD checkpoint and matching Flux VAE.",
+  "model_url": "https://huggingface.co/nvidia/PiD",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Gemma 2 2B ELM Text Encoder"
+    },
+    {
+      "id": "tokenizer",
+      "required": true,
+      "label": "Gemma 2 tokenizer.json"
+    },
+    {
+      "id": "vae",
+      "required": true
+    }
+  ],
+  "context": {
+    "vae_format": "flux"
+  },
+  "defaults": {
+    "sampling_method": "lcm",
+    "scheduler": "lcm",
+    "cfg_scale": 1,
+    "steps": 4,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/pixart-alpha-1024.json b/examples/server/recipes/pixart-alpha-1024.json
new file mode 100644
index 0000000000..1144f87b2a
--- /dev/null
+++ b/examples/server/recipes/pixart-alpha-1024.json
@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "id": "pixart-alpha-1024",
+  "name": "PixArt Alpha XL 1024",
+  "description": "Use the matching PixArt Alpha transformer and SD 1.x-style VAE. Resolution/aspect-ratio micro-conditioning is not currently applied by the runtime.",
+  "model_url": "https://huggingface.co/PixArt-alpha/PixArt-XL-2-1024-MS",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "context": {
+    "model_args": "pixart_vae_scale_factor=0.18215"
+  },
+  "defaults": {
+    "scheduler": "discrete",
+    "sampling_method": "euler",
+    "cfg_scale": 4.5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  }
+}
diff --git a/examples/server/recipes/pixart-alpha-512.json b/examples/server/recipes/pixart-alpha-512.json
new file mode 100644
index 0000000000..804a7a0c9e
--- /dev/null
+++ b/examples/server/recipes/pixart-alpha-512.json
@@ -0,0 +1,42 @@
+{
+  "schema_version": 1,
+  "id": "pixart-alpha-512",
+  "name": "PixArt Alpha XL 512",
+  "description": "Use the matching PixArt Alpha transformer and SD 1.x-style VAE. Resolution/aspect-ratio micro-conditioning is not currently applied by the runtime.",
+  "model_url": "https://huggingface.co/PixArt-alpha/PixArt-XL-2-512x512",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "context": {
+    "model_args": "pixart_vae_scale_factor=0.18215,pixart_pos_embed_base_size=32,pixart_interpolation_scale=1"
+  },
+  "defaults": {
+    "scheduler": "discrete",
+    "sampling_method": "euler",
+    "cfg_scale": 4.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512
+  }
+}
diff --git a/examples/server/recipes/pixart-sigma.json b/examples/server/recipes/pixart-sigma.json
new file mode 100644
index 0000000000..ed4d2f0dec
--- /dev/null
+++ b/examples/server/recipes/pixart-sigma.json
@@ -0,0 +1,37 @@
+{
+  "schema_version": 1,
+  "id": "pixart-sigma",
+  "name": "PixArt Sigma XL 1024",
+  "model_url": "https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 4.5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024,
+    "sampling_method": "euler"
+  }
+}
diff --git a/examples/server/recipes/qwen-image-21-turbo.json b/examples/server/recipes/qwen-image-21-turbo.json
new file mode 100644
index 0000000000..2ef9fa183e
--- /dev/null
+++ b/examples/server/recipes/qwen-image-21-turbo.json
@@ -0,0 +1,60 @@
+{
+  "schema_version": 1,
+  "id": "qwen-image-21-turbo",
+  "name": "Qwen Image 2.1 Turbo",
+  "description": "Use the standalone Turbo diffusion weights for generation and editing with the fixed 8-step schedule. Select the Qwen Image 2.1 VAE or TAEQI2.1, and supply a vision projector when needed.",
+  "model_url": "https://huggingface.co/Qwen/Qwen-Image-2.1-Turbo",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 8B Text Encoder"
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Qwen Image 2.1 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false,
+      "label": "TAEQI2.1"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 8,
+    "width": 2048,
+    "height": 2048,
+    "custom_sigmas": [
+      1.0,
+      0.978453,
+      0.95418,
+      0.926626,
+      0.89508,
+      0.845148,
+      0.704534,
+      0.414568,
+      0.0
+    ]
+  },
+  "features": [
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/qwen-image-21.json b/examples/server/recipes/qwen-image-21.json
new file mode 100644
index 0000000000..4b6c67b86e
--- /dev/null
+++ b/examples/server/recipes/qwen-image-21.json
@@ -0,0 +1,50 @@
+{
+  "schema_version": 1,
+  "id": "qwen-image-21",
+  "name": "Qwen Image 2.1",
+  "description": "Supports generation and editing. Select the full Qwen Image 2.1 VAE or TAEQI2.1. Supply a separate vision projector when needed. Select acceleration LoRAs and adjust their settings separately.",
+  "model_url": "https://huggingface.co/Qwen/Qwen-Image-2.1",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 8B Text Encoder"
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Qwen Image 2.1 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false,
+      "label": "TAEQI2.1"
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 20,
+    "width": 2048,
+    "height": 2048
+  },
+  "features": [
+    "flow_shift",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/qwen-image-edit-2509.json b/examples/server/recipes/qwen-image-edit-2509.json
new file mode 100644
index 0000000000..4c647a0420
--- /dev/null
+++ b/examples/server/recipes/qwen-image-edit-2509.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "qwen-image-edit-2509",
+  "name": "Qwen Image Edit 2509",
+  "description": "Supply reference images. Select a vision projector for GGUF text encoders; omit it if vision weights are included in the text encoder.",
+  "model_url": "https://huggingface.co/Qwen/Qwen-Image-Edit-2509",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Qwen Image VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "flow_shift": 3.0,
+    "cfg_scale": 4,
+    "steps": 20
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/qwen-image-edit-2511.json b/examples/server/recipes/qwen-image-edit-2511.json
new file mode 100644
index 0000000000..cdd16db6cc
--- /dev/null
+++ b/examples/server/recipes/qwen-image-edit-2511.json
@@ -0,0 +1,52 @@
+{
+  "schema_version": 1,
+  "id": "qwen-image-edit-2511",
+  "name": "Qwen Image Edit 2511",
+  "description": "Supply reference images. Select a vision projector for GGUF text encoders; omit it if vision weights are included in the text encoder.",
+  "model_url": "https://huggingface.co/Qwen/Qwen-Image-Edit-2511",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Qwen Image VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "context": {
+    "model_args": "qwen_image_zero_cond_t=true"
+  },
+  "defaults": {
+    "sampling_method": "euler",
+    "flow_shift": 3.0,
+    "cfg_scale": 4,
+    "steps": 20
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/qwen-image-edit.json b/examples/server/recipes/qwen-image-edit.json
new file mode 100644
index 0000000000..5610a6c428
--- /dev/null
+++ b/examples/server/recipes/qwen-image-edit.json
@@ -0,0 +1,49 @@
+{
+  "schema_version": 1,
+  "id": "qwen-image-edit",
+  "name": "Qwen Image Edit",
+  "description": "Supply reference images. Select a vision projector for GGUF text encoders; omit it if vision weights are included in the text encoder.",
+  "model_url": "https://huggingface.co/Qwen/Qwen-Image-Edit",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "llm_vision",
+      "required": false,
+      "label": "Vision projector (required for separate GGUF text encoders)"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Qwen Image VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "flow_shift": 3.0,
+    "cfg_scale": 4,
+    "steps": 20
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/qwen-image-layered.json b/examples/server/recipes/qwen-image-layered.json
new file mode 100644
index 0000000000..96863be313
--- /dev/null
+++ b/examples/server/recipes/qwen-image-layered.json
@@ -0,0 +1,36 @@
+{
+  "schema_version": 1,
+  "id": "qwen-image-layered",
+  "name": "Qwen Image Layered",
+  "description": "Supply a reference image and use the dedicated layered VAE.",
+  "model_url": "https://huggingface.co/Qwen/Qwen-Image-Layered",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": true,
+      "label": "Qwen Image Layered VAE"
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "flow_shift": 3.0,
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 640,
+    "height": 640
+  },
+  "features": [
+    "flow_shift",
+    "img_cfg",
+    "ref_images"
+  ]
+}
diff --git a/examples/server/recipes/qwen-image.json b/examples/server/recipes/qwen-image.json
new file mode 100644
index 0000000000..417a621ae2
--- /dev/null
+++ b/examples/server/recipes/qwen-image.json
@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "id": "qwen-image",
+  "name": "Qwen Image",
+  "description": "Supports text-to-image and image-to-image; pruned transformer variants use the same components.",
+  "model_url": "https://huggingface.co/Qwen/Qwen-Image",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 2.5 VL 7B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Qwen Image VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "flow_shift": 3.0,
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 1328,
+    "height": 1328
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/sd-turbo.json b/examples/server/recipes/sd-turbo.json
new file mode 100644
index 0000000000..7723c81071
--- /dev/null
+++ b/examples/server/recipes/sd-turbo.json
@@ -0,0 +1,36 @@
+{
+  "schema_version": 1,
+  "id": "sd-turbo",
+  "name": "SD Turbo",
+  "description": "Select a single-file SD-Turbo checkpoint. Uses one step with CFG disabled (scale 1 in sd.cpp); use more steps for partial image-to-image denoising.",
+  "model_url": "https://huggingface.co/stabilityai/sd-turbo",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "scheduler": "discrete",
+    "cfg_scale": 1,
+    "steps": 1,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sd1-checkpoint.json b/examples/server/recipes/sd1-checkpoint.json
new file mode 100644
index 0000000000..38d2386775
--- /dev/null
+++ b/examples/server/recipes/sd1-checkpoint.json
@@ -0,0 +1,35 @@
+{
+  "schema_version": 1,
+  "id": "sd1-checkpoint",
+  "name": "SD 1.x",
+  "description": "Supports SD 1.x, compatible LCM checkpoints and layer-count variants. For LCM weights, select LCM sampling and adjust steps/CFG. Tiny SD Diffusers directories must first be converted into a single-file checkpoint.",
+  "model_url": "https://huggingface.co/CompVis/stable-diffusion-v1-4",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "label": "VAE override",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 7.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sd1-controlnet.json b/examples/server/recipes/sd1-controlnet.json
new file mode 100644
index 0000000000..eeaee22fb3
--- /dev/null
+++ b/examples/server/recipes/sd1-controlnet.json
@@ -0,0 +1,38 @@
+{
+  "schema_version": 1,
+  "id": "sd1-controlnet",
+  "name": "SD 1.5 ControlNet",
+  "description": "Supply a control image; input image and mask are optional for image-to-image or inpainting.",
+  "model_url": "https://huggingface.co/lllyasviel/control_v11p_sd15_canny",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "control_net",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 7.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sd1-inpainting.json b/examples/server/recipes/sd1-inpainting.json
new file mode 100644
index 0000000000..f9f3064f3f
--- /dev/null
+++ b/examples/server/recipes/sd1-inpainting.json
@@ -0,0 +1,34 @@
+{
+  "schema_version": 1,
+  "id": "sd1-inpainting",
+  "name": "SD 1.5 Inpainting",
+  "description": "Select an inpainting checkpoint, then supply an input image and mask.",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 7.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "img_cfg"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sd1-ip-adapter.json b/examples/server/recipes/sd1-ip-adapter.json
new file mode 100644
index 0000000000..6c0ab1a6b0
--- /dev/null
+++ b/examples/server/recipes/sd1-ip-adapter.json
@@ -0,0 +1,43 @@
+{
+  "schema_version": 1,
+  "id": "sd1-ip-adapter",
+  "name": "SD 1.5 IP-Adapter / Plus",
+  "description": "Select matching SD 1.5 IP-Adapter or Plus weights and CLIP Vision H, then supply an IP-Adapter image.",
+  "model_url": "https://huggingface.co/h94/IP-Adapter",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "clip_vision",
+      "required": true,
+      "label": "CLIP Vision H"
+    },
+    {
+      "id": "ip_adapter",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 7.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sd2-512-checkpoint.json b/examples/server/recipes/sd2-512-checkpoint.json
new file mode 100644
index 0000000000..77838c2805
--- /dev/null
+++ b/examples/server/recipes/sd2-512-checkpoint.json
@@ -0,0 +1,33 @@
+{
+  "schema_version": 1,
+  "id": "sd2-512-checkpoint",
+  "name": "SD 2.1 512",
+  "description": "Supports SD 2.x 512 and compatible layer-count variants. Tiny SD2 Diffusers directories must first be converted into a single-file checkpoint.",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 7.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sd2-768-checkpoint.json b/examples/server/recipes/sd2-768-checkpoint.json
new file mode 100644
index 0000000000..808b187d7a
--- /dev/null
+++ b/examples/server/recipes/sd2-768-checkpoint.json
@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "id": "sd2-768-checkpoint",
+  "name": "SD 2.1 768",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 7.5,
+    "steps": 20,
+    "width": 768,
+    "height": 768
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sd2-inpainting.json b/examples/server/recipes/sd2-inpainting.json
new file mode 100644
index 0000000000..4980f7f0a5
--- /dev/null
+++ b/examples/server/recipes/sd2-inpainting.json
@@ -0,0 +1,36 @@
+{
+  "schema_version": 1,
+  "id": "sd2-inpainting",
+  "name": "SD 2 Inpainting",
+  "description": "Select an inpainting checkpoint, then supply Init Image and Mask Image.",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "scheduler": "discrete",
+    "sampling_method": "euler_a",
+    "cfg_scale": 7.5,
+    "steps": 20,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "img_cfg"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sd3-medium.json b/examples/server/recipes/sd3-medium.json
new file mode 100644
index 0000000000..7c0e8fbffd
--- /dev/null
+++ b/examples/server/recipes/sd3-medium.json
@@ -0,0 +1,33 @@
+{
+  "schema_version": 1,
+  "id": "sd3-medium",
+  "name": "SD 3 Medium",
+  "description": "Checkpoint with CLIP-L, CLIP-G and T5 included.",
+  "model_url": "https://huggingface.co/stabilityai/stable-diffusion-3-medium-diffusers",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 7,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "clip_skip",
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/sd35-large.json b/examples/server/recipes/sd35-large.json
new file mode 100644
index 0000000000..a231db99c6
--- /dev/null
+++ b/examples/server/recipes/sd35-large.json
@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "id": "sd35-large",
+  "name": "SD 3.5 Large",
+  "model_url": "https://huggingface.co/stabilityai/stable-diffusion-3.5-large",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "clip_g",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 3.5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "clip_skip",
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/sd35-medium.json b/examples/server/recipes/sd35-medium.json
new file mode 100644
index 0000000000..75080253aa
--- /dev/null
+++ b/examples/server/recipes/sd35-medium.json
@@ -0,0 +1,44 @@
+{
+  "schema_version": 1,
+  "id": "sd35-medium",
+  "name": "SD 3.5 Medium",
+  "model_url": "https://huggingface.co/stabilityai/stable-diffusion-3.5-medium",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "clip_l",
+      "required": true
+    },
+    {
+      "id": "clip_g",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 4.5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "clip_skip",
+    "flow_shift",
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/sdxl-checkpoint.json b/examples/server/recipes/sdxl-checkpoint.json
new file mode 100644
index 0000000000..372d5a6d4a
--- /dev/null
+++ b/examples/server/recipes/sdxl-checkpoint.json
@@ -0,0 +1,36 @@
+{
+  "schema_version": 1,
+  "id": "sdxl-checkpoint",
+  "name": "SDXL",
+  "description": "Supports SDXL and compatible reduced-depth checkpoints such as SSD-1B and Segmind Vega.",
+  "model_url": "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "label": "VAE override",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "shifted_timestep"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sdxl-controlnet.json b/examples/server/recipes/sdxl-controlnet.json
new file mode 100644
index 0000000000..c50efbd2b4
--- /dev/null
+++ b/examples/server/recipes/sdxl-controlnet.json
@@ -0,0 +1,41 @@
+{
+  "schema_version": 1,
+  "id": "sdxl-controlnet",
+  "name": "SDXL ControlNet",
+  "description": "Select a compatible SDXL ControlNet and supply a preprocessed control image. Input image and mask are optional for image-to-image or inpainting.",
+  "model_url": "https://huggingface.co/diffusers/controlnet-canny-sdxl-1.0",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "control_net",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "scheduler": "discrete",
+    "sampling_method": "euler_a",
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "shifted_timestep"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sdxl-inpainting.json b/examples/server/recipes/sdxl-inpainting.json
new file mode 100644
index 0000000000..574423d08f
--- /dev/null
+++ b/examples/server/recipes/sdxl-inpainting.json
@@ -0,0 +1,39 @@
+{
+  "schema_version": 1,
+  "id": "sdxl-inpainting",
+  "name": "SDXL Inpainting",
+  "description": "Select an inpainting checkpoint, then supply Init Image and Mask Image.",
+  "model_url": "https://huggingface.co/diffusers/stable-diffusion-xl-1.0-inpainting-0.1",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "label": "VAE override",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "scheduler": "discrete",
+    "sampling_method": "euler_a",
+    "cfg_scale": 8,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "shifted_timestep",
+    "img_cfg"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sdxl-ip-adapter.json b/examples/server/recipes/sdxl-ip-adapter.json
new file mode 100644
index 0000000000..b649fb1d5a
--- /dev/null
+++ b/examples/server/recipes/sdxl-ip-adapter.json
@@ -0,0 +1,46 @@
+{
+  "schema_version": 1,
+  "id": "sdxl-ip-adapter",
+  "name": "SDXL IP-Adapter / Plus",
+  "description": "Select matching SDXL ViT-H IP-Adapter or Plus weights and CLIP Vision H, then supply an IP-Adapter image.",
+  "model_url": "https://huggingface.co/h94/IP-Adapter",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "clip_vision",
+      "required": true,
+      "label": "CLIP Vision H"
+    },
+    {
+      "id": "ip_adapter",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "scheduler": "discrete",
+    "sampling_method": "dpm++2m",
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "shifted_timestep"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sdxl-photomaker.json b/examples/server/recipes/sdxl-photomaker.json
new file mode 100644
index 0000000000..025fb7668e
--- /dev/null
+++ b/examples/server/recipes/sdxl-photomaker.json
@@ -0,0 +1,37 @@
+{
+  "schema_version": 1,
+  "id": "sdxl-photomaker",
+  "name": "SDXL PhotoMaker",
+  "description": "Select PhotoMaker v1 and supply identity images. Include the img trigger in the prompt.",
+  "model_url": "https://huggingface.co/TencentARC/PhotoMaker",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "photo_maker",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "shifted_timestep"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sdxl-turbo.json b/examples/server/recipes/sdxl-turbo.json
new file mode 100644
index 0000000000..8650e6c581
--- /dev/null
+++ b/examples/server/recipes/sdxl-turbo.json
@@ -0,0 +1,34 @@
+{
+  "schema_version": 1,
+  "id": "sdxl-turbo",
+  "name": "SDXL Turbo",
+  "model_url": "https://huggingface.co/stabilityai/sdxl-turbo",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 1,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip",
+    "shifted_timestep"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sdxs.json b/examples/server/recipes/sdxs.json
new file mode 100644
index 0000000000..19e218ac5c
--- /dev/null
+++ b/examples/server/recipes/sdxs.json
@@ -0,0 +1,29 @@
+{
+  "schema_version": 1,
+  "id": "sdxs",
+  "name": "SDXS",
+  "model_url": "https://huggingface.co/IDKiro/sdxs-512-dreamshaper",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "required": false
+    }
+  ],
+  "defaults": {
+    "cfg_scale": 1,
+    "steps": 1,
+    "width": 512,
+    "height": 512
+  },
+  "features": [
+    "cache_unet",
+    "clip_skip"
+  ],
+  "disabled_features": [
+    "cache_dit"
+  ]
+}
diff --git a/examples/server/recipes/sefi-image.json b/examples/server/recipes/sefi-image.json
new file mode 100644
index 0000000000..35e933360d
--- /dev/null
+++ b/examples/server/recipes/sefi-image.json
@@ -0,0 +1,39 @@
+{
+  "schema_version": 1,
+  "id": "sefi-image",
+  "name": "SeFi Image 5B RL",
+  "model_url": "https://huggingface.co/SeFi-Image/SeFi-Image-5B-Base",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 VL 4B Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 4,
+    "steps": 20,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "slg"
+  ]
+}
diff --git a/examples/server/recipes/sensenova-u15.json b/examples/server/recipes/sensenova-u15.json
new file mode 100644
index 0000000000..afd20466b6
--- /dev/null
+++ b/examples/server/recipes/sensenova-u15.json
@@ -0,0 +1,24 @@
+{
+  "schema_version": 1,
+  "id": "sensenova-u15",
+  "name": "SenseNova U1.5",
+  "description": "Bundled MoT checkpoint.",
+  "model_url": "https://huggingface.co/sensenova/SenseNova-U1.5-8B-MoT",
+  "slots": [
+    {
+      "id": "model",
+      "required": true
+    }
+  ],
+  "defaults": {
+    "flow_shift": 3.0,
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 2048,
+    "height": 2048,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/wan21-flf2v.json b/examples/server/recipes/wan21-flf2v.json
new file mode 100644
index 0000000000..02f0e80a91
--- /dev/null
+++ b/examples/server/recipes/wan21-flf2v.json
@@ -0,0 +1,56 @@
+{
+  "schema_version": 1,
+  "id": "wan21-flf2v",
+  "name": "Wan 2.1 FLF2V 14B",
+  "description": "Select FLF2V weights and supply both start and end images.",
+  "model_url": "https://huggingface.co/Wan-AI/Wan2.1-FLF2V-14B-720P",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true,
+      "label": "UMT5-XXL"
+    },
+    {
+      "id": "clip_vision",
+      "required": true,
+      "label": "CLIP Vision H"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Wan 2.1 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "flow_shift": 16,
+    "video_frames": 21,
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1280,
+    "height": 720
+  },
+  "features": [
+    "end_image",
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/wan21-i2v.json b/examples/server/recipes/wan21-i2v.json
new file mode 100644
index 0000000000..d8d8d13ee1
--- /dev/null
+++ b/examples/server/recipes/wan21-i2v.json
@@ -0,0 +1,56 @@
+{
+  "schema_version": 1,
+  "id": "wan21-i2v",
+  "name": "Wan 2.1 I2V",
+  "description": "Image-to-video with Wan 2.1, compatible SkyReels V2 weights, or a single Wan 2.2 I2V expert. CLIP Vision is required by Wan 2.1 I2V and is unused by Wan 2.2. Defaults target Wan 2.1 480p.",
+  "model_url": "https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "label": "UMT5-XXL",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "label": "Wan 2.1 VAE",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    },
+    {
+      "id": "clip_vision",
+      "label": "CLIP Vision H (Wan 2.1 I2V)",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "fps": 16,
+    "flow_shift": 3,
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 832,
+    "height": 480
+  },
+  "features": [
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/wan21-t2v.json b/examples/server/recipes/wan21-t2v.json
new file mode 100644
index 0000000000..367611c413
--- /dev/null
+++ b/examples/server/recipes/wan21-t2v.json
@@ -0,0 +1,50 @@
+{
+  "schema_version": 1,
+  "id": "wan21-t2v",
+  "name": "Wan 2.1 T2V",
+  "description": "Text-to-video with Wan 2.1 or a compatible single Wan 2.2 T2V expert. Defaults target Wan 2.1 14B; adjust settings for the selected weights and acceleration LoRAs.",
+  "model_url": "https://huggingface.co/Wan-AI/Wan2.1-T2V-14B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "label": "UMT5-XXL",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "label": "Wan 2.1 VAE",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "fps": 16,
+    "flow_shift": 5,
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1280,
+    "height": 720
+  },
+  "features": [
+    "flow_shift"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/wan21-vace.json b/examples/server/recipes/wan21-vace.json
new file mode 100644
index 0000000000..7b7f7e41b8
--- /dev/null
+++ b/examples/server/recipes/wan21-vace.json
@@ -0,0 +1,52 @@
+{
+  "schema_version": 1,
+  "id": "wan21-vace",
+  "name": "Wan 2.1 VACE 1.3B / 14B",
+  "description": "Select matching VACE weights. Supports text, reference images and control video through Video Generation.",
+  "model_url": "https://huggingface.co/Wan-AI/Wan2.1-VACE-14B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true,
+      "label": "UMT5-XXL"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Wan 2.1 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 832,
+    "height": 480,
+    "flow_shift": 16
+  },
+  "features": [
+    "control_frames",
+    "flow_shift",
+    "init_image",
+    "vace"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/wan22-i2v.json b/examples/server/recipes/wan22-i2v.json
new file mode 100644
index 0000000000..62afdfce4f
--- /dev/null
+++ b/examples/server/recipes/wan22-i2v.json
@@ -0,0 +1,62 @@
+{
+  "schema_version": 1,
+  "id": "wan22-i2v",
+  "name": "Wan 2.2 I2V A14B",
+  "description": "Image-to-video. Select a start image in Video Generation after loading. LightX2V acceleration LoRAs must match the I2V low- and high-noise models; configure LoRAs and sampling separately.",
+  "model_url": "https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "label": "Low-noise diffusion model",
+      "required": true
+    },
+    {
+      "id": "high_noise_diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "label": "UMT5-XXL",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "label": "Wan 2.1 VAE",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "fps": 16,
+    "flow_shift": 5,
+    "high_noise_steps": -1,
+    "high_noise_cfg_scale": 3.5,
+    "high_noise_sampling_method": "euler",
+    "high_noise_flow_shift": 5,
+    "cfg_scale": 3.5,
+    "steps": 20,
+    "width": 1280,
+    "height": 720,
+    "moe_boundary": 0.9
+  },
+  "features": [
+    "end_image",
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/wan22-s2v.json b/examples/server/recipes/wan22-s2v.json
new file mode 100644
index 0000000000..ba1ee4f7c3
--- /dev/null
+++ b/examples/server/recipes/wan22-s2v.json
@@ -0,0 +1,53 @@
+{
+  "schema_version": 1,
+  "id": "wan22-s2v",
+  "name": "Wan 2.2 S2V 14B",
+  "description": "Supply a start image and reference audio through Video Generation.",
+  "model_url": "https://huggingface.co/Wan-AI/Wan2.2-S2V-14B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true,
+      "label": "UMT5-XXL"
+    },
+    {
+      "id": "audio_encoder",
+      "required": true,
+      "label": "Wav2Vec2 Large English"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Wan 2.1 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "flow_shift": 3,
+    "video_frames": 33,
+    "cfg_scale": 4.5,
+    "steps": 20
+  },
+  "features": [
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/wan22-t2v.json b/examples/server/recipes/wan22-t2v.json
new file mode 100644
index 0000000000..905bcfa4ff
--- /dev/null
+++ b/examples/server/recipes/wan22-t2v.json
@@ -0,0 +1,60 @@
+{
+  "schema_version": 1,
+  "id": "wan22-t2v",
+  "name": "Wan 2.2 T2V A14B",
+  "description": "Text-to-video. Use Video Generation after loading. LightX2V acceleration LoRAs must match the T2V low- and high-noise models; configure LoRAs and sampling separately.",
+  "model_url": "https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "label": "Low-noise diffusion model",
+      "required": true
+    },
+    {
+      "id": "high_noise_diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "label": "UMT5-XXL",
+      "required": true
+    },
+    {
+      "id": "vae",
+      "label": "Wan 2.1 VAE",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "fps": 16,
+    "flow_shift": 12,
+    "high_noise_steps": -1,
+    "high_noise_cfg_scale": 4,
+    "high_noise_sampling_method": "euler",
+    "high_noise_flow_shift": 12,
+    "cfg_scale": 3,
+    "steps": 20,
+    "width": 1280,
+    "height": 720,
+    "moe_boundary": 0.875
+  },
+  "features": [
+    "flow_shift"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/wan22-ti2v-5b.json b/examples/server/recipes/wan22-ti2v-5b.json
new file mode 100644
index 0000000000..3e2aa35b88
--- /dev/null
+++ b/examples/server/recipes/wan22-ti2v-5b.json
@@ -0,0 +1,50 @@
+{
+  "schema_version": 1,
+  "id": "wan22-ti2v-5b",
+  "name": "Wan 2.2 TI2V 5B",
+  "description": "Use Wan 2.2 VAE. Supports text, start-image and start/end-image conditioning.",
+  "model_url": "https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "t5xxl",
+      "required": true,
+      "label": "UMT5-XXL"
+    },
+    {
+      "id": "vae",
+      "required": false,
+      "label": "Wan 2.2 VAE"
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "video_frames": 33,
+    "cfg_scale": 5,
+    "steps": 20,
+    "width": 1280,
+    "height": 704,
+    "flow_shift": 5
+  },
+  "features": [
+    "flow_shift",
+    "init_image"
+  ],
+  "disabled_features": [
+    "img2img",
+    "mask_image"
+  ]
+}
diff --git a/examples/server/recipes/z-image-l2p.json b/examples/server/recipes/z-image-l2p.json
new file mode 100644
index 0000000000..0375f1387b
--- /dev/null
+++ b/examples/server/recipes/z-image-l2p.json
@@ -0,0 +1,29 @@
+{
+  "schema_version": 1,
+  "id": "z-image-l2p",
+  "name": "Z-Image L2P",
+  "description": "Pixel-space L2P checkpoint with Qwen 3 4B; no VAE is needed. Use dimensions divisible by 16.",
+  "model_url": "https://huggingface.co/zhen-nan/L2P",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 4B Text Encoder"
+    }
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "scheduler": "discrete"
+  },
+  "features": [
+    "flow_shift"
+  ],
+  "disabled_features": [
+    "vae_tiling",
+    "preview"
+  ]
+}
diff --git a/examples/server/recipes/z-image-turbo.json b/examples/server/recipes/z-image-turbo.json
new file mode 100644
index 0000000000..99737674ee
--- /dev/null
+++ b/examples/server/recipes/z-image-turbo.json
@@ -0,0 +1,41 @@
+{
+  "schema_version": 1,
+  "id": "z-image-turbo",
+  "name": "Z-Image Turbo",
+  "model_url": "https://huggingface.co/Tongyi-MAI/Z-Image-Turbo",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "sampling_method": "euler",
+    "cfg_scale": 1,
+    "steps": 9,
+    "width": 1024,
+    "height": 1024
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes/z-image.json b/examples/server/recipes/z-image.json
new file mode 100644
index 0000000000..bcf2da7cd3
--- /dev/null
+++ b/examples/server/recipes/z-image.json
@@ -0,0 +1,41 @@
+{
+  "schema_version": 1,
+  "id": "z-image",
+  "name": "Z-Image",
+  "model_url": "https://huggingface.co/Tongyi-MAI/Z-Image",
+  "slots": [
+    {
+      "id": "diffusion_model",
+      "required": true
+    },
+    {
+      "id": "llm",
+      "required": true,
+      "label": "Qwen 3 Text Encoder"
+    },
+    {
+      "id": "vae",
+      "required": false
+    },
+    {
+      "id": "tae",
+      "required": false
+    }
+  ],
+  "required_any": [
+    [
+      "vae",
+      "tae"
+    ]
+  ],
+  "defaults": {
+    "cfg_scale": 4,
+    "steps": 20,
+    "width": 720,
+    "height": 1280,
+    "sampling_method": "euler"
+  },
+  "features": [
+    "flow_shift"
+  ]
+}
diff --git a/examples/server/recipes_builtin.h.in b/examples/server/recipes_builtin.h.in
new file mode 100644
index 0000000000..ff31b4d2c4
--- /dev/null
+++ b/examples/server/recipes_builtin.h.in
@@ -0,0 +1,7 @@
+#ifndef __SD_EXAMPLES_SERVER_RECIPES_BUILTIN_H__
+#define __SD_EXAMPLES_SERVER_RECIPES_BUILTIN_H__
+
+static const char* const k_builtin_recipes[] = {
+@SD_BUILTIN_RECIPES@};
+
+#endif
diff --git a/examples/server/routes.h b/examples/server/routes.h
index 1a4efb5b70..f77939184e 100644
--- a/examples/server/routes.h
+++ b/examples/server/routes.h
@@ -9,3 +9,18 @@ void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_param
 void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt);
 void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt);
 void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt);
+void register_model_endpoints(httplib::Server& svr, ServerRuntime& rt);
+
+class RuntimeRoutes {
+    httplib::Server& server_;
+    ServerRuntime& runtime_;
+
+    httplib::Server::Handler protect(httplib::Server::Handler handler);
+
+public:
+    RuntimeRoutes(httplib::Server& server, ServerRuntime& runtime)
+        : server_(server), runtime_(runtime) {}
+
+    void get(const std::string& pattern, httplib::Server::Handler handler);
+    void post(const std::string& pattern, httplib::Server::Handler handler);
+};
diff --git a/examples/server/routes_index.cpp b/examples/server/routes_index.cpp
index 1341ff84ac..14ad814b49 100644
--- a/examples/server/routes_index.cpp
+++ b/examples/server/routes_index.cpp
@@ -1,5 +1,6 @@
 #include "routes.h"
 
+#include <filesystem>
 #include <fstream>
 #include <iterator>
 
@@ -7,7 +8,7 @@ void register_index_endpoints(httplib::Server& svr, const SDSvrParams& svr_param
     const std::string serve_html_path = svr_params.serve_html_path;
     svr.Get("/", [serve_html_path, index_html](const httplib::Request&, httplib::Response& res) {
         if (!serve_html_path.empty()) {
-            std::ifstream file(serve_html_path);
+            std::ifstream file(std::filesystem::u8path(serve_html_path));
             if (file) {
                 std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
                 res.set_content(content, "text/html");
diff --git a/examples/server/routes_models.cpp b/examples/server/routes_models.cpp
new file mode 100644
index 0000000000..ea14eb821a
--- /dev/null
+++ b/examples/server/routes_models.cpp
@@ -0,0 +1,150 @@
+#include "routes.h"
+
+#include "async_jobs.h"
+#include "model_catalog.h"
+
+static void model_error(httplib::Response& res, int status, const std::string& code, const std::string& message) {
+    res.status = status;
+    res.set_content(json({{"error", message}, {"code", code}}).dump(), "application/json");
+}
+
+httplib::Server::Handler RuntimeRoutes::protect(httplib::Server::Handler handler) {
+    ServerRuntime* runtime = &runtime_;
+    return [runtime, handler](const httplib::Request& req, httplib::Response& res) {
+        std::shared_lock<std::shared_mutex> lock(runtime->models->runtime_mutex, std::try_to_lock);
+        if (!lock.owns_lock()) {
+            model_error(res, 503, "model_loading", "Model is loading or unloading");
+            return;
+        }
+        const bool generation = req.path == "/sdcpp/v1/img_gen" || req.path == "/sdcpp/v1/vid_gen" ||
+                                req.path == "/sdapi/v1/txt2img" || req.path == "/sdapi/v1/img2img" ||
+                                req.path == "/v1/images/generations" || req.path == "/v1/images/edits";
+        if (generation && !runtime->sd_ctx) {
+            model_error(res, 503, "model_not_loaded", "No model is loaded");
+            return;
+        }
+        handler(req, res);
+    };
+}
+
+void RuntimeRoutes::get(const std::string& pattern, httplib::Server::Handler handler) {
+    // Job polling and cancellation only access the job manager, even during a model transition.
+    server_.Get(pattern, pattern.find("/sdcpp/v1/jobs/") != std::string::npos ? handler : protect(handler));
+}
+
+void RuntimeRoutes::post(const std::string& pattern, httplib::Server::Handler handler) {
+    server_.Post(pattern, pattern.find("/sdcpp/v1/jobs/") != std::string::npos ? handler : protect(handler));
+}
+
+static void set_model_status(ServerRuntime& runtime, const char* state, const json& recipe, const json& selection, const json& error = nullptr) {
+    std::lock_guard<std::mutex> lock(runtime.models->status_mutex);
+    auto& models            = *runtime.models;
+    const uint64_t revision = models.status.value("revision", uint64_t(0)) + 1;
+    json elapsed            = nullptr;
+    if (!recipe.is_null()) {
+        const auto now = std::chrono::steady_clock::now();
+        if (std::string(state) == "loading") {
+            models.load_started_at = now;
+            elapsed                = 0.0;
+        } else {
+            elapsed = std::chrono::duration<double>(now - models.load_started_at).count();
+        }
+    }
+    models.status = {{"state", state}, {"recipe", recipe}, {"models", selection}, {"error", error}, {"revision", revision}, {"load_elapsed_seconds", elapsed}};
+}
+
+static void change_model(ServerRuntime& runtime, const httplib::Request& req, httplib::Response& res, bool unload) {
+    auto& models = *runtime.models;
+    std::unique_lock<std::shared_mutex> lock(models.runtime_mutex, std::try_to_lock);
+    if (!lock.owns_lock()) {
+        model_error(res, 409, "model_busy", "Model is in use or another model operation is running");
+        return;
+    }
+    {
+        // Admission holds runtime_mutex through enqueue. Jobs stay pending until
+        // generation and metadata encoding finish, so no worker can retain this context here.
+        std::lock_guard<std::mutex> jobs_lock(runtime.async_job_manager->mutex);
+        if (count_pending_jobs(*runtime.async_job_manager) != 0) {
+            model_error(res, 409, "model_busy", "Finish or cancel pending generation jobs before switching models");
+            return;
+        }
+    }
+
+    json body;
+    json loaded_recipe;
+    SDContextParams context;
+    SDGenerationParams defaults;
+    if (!unload) {
+        try {
+            body = json::parse(req.body);
+            prepare_model_load(runtime, body, context, defaults, &loaded_recipe);
+        } catch (const std::exception& e) {
+            model_error(res, 400, "invalid_model_config", e.what());
+            return;
+        }
+    }
+
+    const json recipe    = unload ? json(nullptr) : body.at("recipe");
+    const json selection = unload ? json::object() : body.at("models");
+    set_model_status(runtime, "loading", recipe, selection);
+    try {
+        runtime.sd_ctx = nullptr;
+        models.context.reset();
+        models.active_recipe        = nullptr;
+        *runtime.ctx_params         = unload ? models.base_context : std::move(context);
+        *runtime.default_gen_params = unload ? models.base_defaults : std::move(defaults);
+        if (!unload) {
+            auto params = runtime.ctx_params->to_sd_ctx_params_t(false);
+            models.context.reset(new_sd_ctx(&params));
+            if (!models.context) {
+                throw std::runtime_error("Model initialization failed; check the server log for component compatibility and memory errors");
+            }
+            runtime.sd_ctx       = models.context.get();
+            models.active_recipe = std::move(loaded_recipe);
+        }
+        set_model_status(runtime, unload ? "empty" : "ready", recipe, selection);
+        res.set_content(model_status(runtime).dump(), "application/json");
+    } catch (const std::exception& e) {
+        runtime.sd_ctx = nullptr;
+        models.context.reset();
+        models.active_recipe        = nullptr;
+        *runtime.ctx_params         = models.base_context;
+        *runtime.default_gen_params = models.base_defaults;
+        set_model_status(runtime, "error", recipe, selection, e.what());
+        model_error(res, 422, "model_load_failed", e.what());
+    }
+}
+
+void register_model_endpoints(httplib::Server& svr, ServerRuntime& runtime) {
+    svr.Get("/sdcpp/v1/recipes", [&runtime](const httplib::Request&, httplib::Response& res) {
+        auto& models = *runtime.models;
+        std::lock_guard<std::mutex> lock(models.catalog_mutex);
+        json items = json::array();
+        for (const auto& entry : models.recipes)
+            items.push_back(entry.second);
+        res.set_content(json({{"recipes", items}, {"errors", models.catalog_errors}}).dump(), "application/json");
+    });
+    svr.Get("/sdcpp/v1/model-files", [&runtime](const httplib::Request&, httplib::Response& res) {
+        auto& models = *runtime.models;
+        std::lock_guard<std::mutex> lock(models.catalog_mutex);
+        json items = json::array();
+        for (const auto& entry : models.files) {
+            items.push_back({{"id", entry.first}, {"name", entry.second.name}, {"category", entry.second.category}, {"root", entry.second.root.id}, {"relative_path", entry.second.relative_path}});
+        }
+        res.set_content(json({{"files", items}, {"errors", models.catalog_errors}}).dump(), "application/json");
+    });
+    svr.Post("/sdcpp/v1/models/refresh", [&runtime](const httplib::Request&, httplib::Response& res) {
+        refresh_model_catalog(runtime);
+        std::lock_guard<std::mutex> lock(runtime.models->catalog_mutex);
+        res.set_content(json({{"errors", runtime.models->catalog_errors}}).dump(), "application/json");
+    });
+    svr.Get("/sdcpp/v1/models/status", [&runtime](const httplib::Request&, httplib::Response& res) {
+        res.set_content(model_status(runtime).dump(), "application/json");
+    });
+    svr.Post("/sdcpp/v1/models/load", [&runtime](const httplib::Request& req, httplib::Response& res) {
+        change_model(runtime, req, res, false);
+    });
+    svr.Post("/sdcpp/v1/models/unload", [&runtime](const httplib::Request& req, httplib::Response& res) {
+        change_model(runtime, req, res, true);
+    });
+}
diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp
index 0f122922ce..29dfe313a7 100644
--- a/examples/server/routes_openai.cpp
+++ b/examples/server/routes_openai.cpp
@@ -7,6 +7,7 @@
 #include "common/common.h"
 #include "common/media_io.h"
 #include "common/resource_owners.hpp"
+#include "openai_edit_policy.h"
 
 static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
     std::regex re("<sd_cpp_extra_args>(.*?)</sd_cpp_extra_args>");
@@ -36,7 +37,7 @@ static bool build_openai_generation_request(const httplib::Request& req,
     std::string output_format = j.value("output_format", "png");
     int output_compression    = j.value("output_compression", 100);
     int width                 = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->width : 512;
-    int height                = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->height : 512;
+    int height                = runtime.default_gen_params->height > 0 ? runtime.default_gen_params->height : 512;
     if (!size.empty()) {
         auto pos = size.find('x');
         if (pos != std::string::npos) {
@@ -157,42 +158,62 @@ static bool build_openai_edit_request(const httplib::Request& req,
     request.gen_params.height      = height;
     request.gen_params.batch_count = n;
 
+    std::string sd_cpp_extra_args_str          = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
+    const json extra_args                      = sd_cpp_extra_args_str.empty()
+                                                     ? json::object()
+                                                     : json::parse(sd_cpp_extra_args_str, nullptr, false);
+    const bool has_extra_init_image_override   = extra_args.is_object() && extra_args.contains("init_image");
+    const bool has_extra_ref_images_override   = extra_args.is_object() && extra_args.contains("ref_images");
+    const bool has_extra_strength_override     = extra_args.is_object() &&
+                                                 extra_args.contains("strength") &&
+                                                 extra_args["strength"].is_number();
+    const bool has_extra_mask_image            = extra_args.is_object() &&
+                                                 extra_args.contains("mask_image") &&
+                                                 !extra_args["mask_image"].is_null();
+    const bool supports_ref_image_conditioning = sd_ctx_supports_ref_image_conditioning(runtime.sd_ctx);
+    const bool may_need_uploaded_init_image    = !has_extra_init_image_override &&
+                                                 (!supports_ref_image_conditioning || !mask_bytes.empty() ||
+                                                  has_extra_mask_image || has_extra_strength_override || has_extra_ref_images_override);
+    SDImageOwner uploaded_image_for_init;
+    bool uploaded_image_was_first_reference = false;
     for (auto& bytes : images_bytes) {
-        int img_w           = 0;
-        int img_h           = 0;
-        uint8_t* raw_pixels = load_image_from_memory(
-            reinterpret_cast<const char*>(bytes.data()),
-            static_cast<int>(bytes.size()),
-            img_w, img_h,
-            width, height, 3);
+        int img_w            = 0;
+        int img_h            = 0;
+        int resolved_channel = 0;
+        uint8_t* raw_pixels  = load_image_from_memory(
+             reinterpret_cast<const char*>(bytes.data()),
+             static_cast<int>(bytes.size()),
+             img_w, img_h, resolved_channel,
+             0, 0,
+             0);
         if (raw_pixels == nullptr) {
             continue;
         }
 
-        SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, 3, raw_pixels});
+        const bool is_first_ref_image = request.gen_params.ref_images.empty();
+        SDImageOwner image_owner({(uint32_t)img_w, (uint32_t)img_h, (uint32_t)resolved_channel, raw_pixels});
         request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height);
-        request.gen_params.ref_images.push_back(std::move(image_owner));
-    }
 
-    if (!request.gen_params.ref_images.empty()) {
-        request.gen_params.init_image = request.gen_params.ref_images.front();
+        if (is_first_ref_image) {
+            uploaded_image_was_first_reference = true;
+            if (may_need_uploaded_init_image) {
+                uploaded_image_for_init = image_owner;
+            }
+        }
+
+        request.gen_params.ref_images.push_back(std::move(image_owner));
     }
 
     if (!mask_bytes.empty()) {
-        int expected_width  = 0;
-        int expected_height = 0;
-        if (request.gen_params.width_and_height_are_set()) {
-            expected_width  = request.gen_params.width;
-            expected_height = request.gen_params.height;
-        }
-        int mask_w = 0;
-        int mask_h = 0;
+        int mask_w       = 0;
+        int mask_h       = 0;
+        int mask_channel = 0;
 
         uint8_t* mask_raw = load_image_from_memory(
             reinterpret_cast<const char*>(mask_bytes.data()),
             static_cast<int>(mask_bytes.size()),
-            mask_w, mask_h,
-            expected_width, expected_height, 1);
+            mask_w, mask_h, mask_channel,
+            0, 0, 1);
         request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw});
         const sd_image_t& mask_image = request.gen_params.mask_image.get();
         request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height);
@@ -205,12 +226,26 @@ static bool build_openai_edit_request(const httplib::Request& req,
         });
     }
 
-    std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt);
     if (!sd_cpp_extra_args_str.empty() && !request.gen_params.from_json_str(sd_cpp_extra_args_str)) {
         error_message = "invalid sd_cpp_extra_args";
         return false;
     }
 
+    const auto init_image_action = apply_openai_edit_init_image_policy(
+        request.gen_params.init_image,
+        uploaded_image_for_init,
+        uploaded_image_was_first_reference,
+        supports_ref_image_conditioning,
+        !request.gen_params.ref_images.empty(),
+        request.gen_params.mask_image.get().data != nullptr,
+        has_extra_init_image_override,
+        has_extra_strength_override);
+    if (init_image_action == OpenAIEditInitImageAction::UseUploaded &&
+        request.gen_params.init_image.get().data == nullptr) {
+        error_message = "could not allocate init image";
+        return false;
+    }
+
     // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
     if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
         error_message = "invalid params";
@@ -246,15 +281,18 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime,
 
 void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
     ServerRuntime* runtime = &rt;
+    RuntimeRoutes routes(svr, rt);
 
-    svr.Get("/v1/models", [runtime](const httplib::Request&, httplib::Response& res) {
+    routes.get("/v1/models", [runtime](const httplib::Request&, httplib::Response& res) {
         json r;
         r["data"] = json::array();
-        r["data"].push_back({{"id", "sd-cpp-local"}, {"object", "model"}, {"owned_by", "local"}});
+        if (runtime->sd_ctx) {
+            r["data"].push_back({{"id", "sd-cpp-local"}, {"object", "model"}, {"owned_by", "local"}});
+        }
         res.set_content(r.dump(), "application/json");
     });
 
-    svr.Post("/v1/images/generations", [runtime](const httplib::Request& req, httplib::Response& res) {
+    routes.post("/v1/images/generations", [runtime](const httplib::Request& req, httplib::Response& res) {
         try {
             if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
                 res.status = 400;
@@ -270,7 +308,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 return;
             }
 
-            LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str());
+            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
 
             SDImageVec results;
             if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
@@ -328,7 +366,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         }
     });
 
-    svr.Post("/v1/images/edits", [runtime](const httplib::Request& req, httplib::Response& res) {
+    routes.post("/v1/images/edits", [runtime](const httplib::Request& req, httplib::Response& res) {
         try {
             if (!runtime_supports_generation_mode(*runtime, IMG_GEN)) {
                 res.status = 400;
@@ -344,7 +382,7 @@ void register_openai_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 return;
             }
 
-            LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str());
+            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
 
             SDImageVec results;
             if (!execute_sync_img_gen_request(*runtime, request, results, error_message)) {
diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp
index 699ba022d8..4b8e7a5b1e 100644
--- a/examples/server/routes_sdapi.cpp
+++ b/examples/server/routes_sdapi.cpp
@@ -10,6 +10,7 @@
 #include "common/common.h"
 #include "common/media_io.h"
 #include "common/resource_owners.hpp"
+#include "model_catalog.h"
 
 namespace fs = std::filesystem;
 
@@ -28,10 +29,10 @@ static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) {
 static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
     const auto& ctx = *runtime.ctx_params;
     if (!ctx.model_path.empty()) {
-        return fs::path(ctx.model_path);
+        return fs::u8path(ctx.model_path);
     }
     if (!ctx.diffusion_model_path.empty()) {
-        return fs::path(ctx.diffusion_model_path);
+        return fs::u8path(ctx.diffusion_model_path);
     }
     return {};
 }
@@ -80,17 +81,6 @@ static enum sample_method_t get_sdapi_sample_method(std::string name) {
     return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;
 }
 
-static void assign_solid_mask(SDImageOwner& mask_owner, int width, int height) {
-    const size_t pixel_count = static_cast<size_t>(width) * static_cast<size_t>(height);
-    uint8_t* raw_mask        = static_cast<uint8_t*>(malloc(pixel_count));
-    if (raw_mask == nullptr) {
-        mask_owner.reset({0, 0, 1, nullptr});
-        return;
-    }
-    std::memset(raw_mask, 255, pixel_count);
-    mask_owner.reset({(uint32_t)width, (uint32_t)height, 1, raw_mask});
-}
-
 static bool build_sdapi_img_gen_request(const json& j,
                                         ServerRuntime& runtime,
                                         bool img2img,
@@ -126,8 +116,9 @@ static bool build_sdapi_img_gen_request(const json& j,
     request.gen_params.sample_params.sample_steps     = steps;
     request.gen_params.batch_count                    = batch_size;
     request.gen_params.sample_params.guidance.txt_cfg = cfg_scale;
-    request.gen_params.width                          = j.value("width", -1);
-    request.gen_params.height                         = j.value("height", -1);
+    const bool recipe_dimensions                      = !img2img && model_status(runtime)["recipe"].is_string();
+    request.gen_params.width                          = j.value("width", recipe_dimensions ? runtime.default_gen_params->width : -1);
+    request.gen_params.height                         = j.value("height", recipe_dimensions ? runtime.default_gen_params->height : -1);
 
     if (!img2img && j.value("enable_hr", false)) {
         request.gen_params.hires_enabled = true;
@@ -162,6 +153,7 @@ static bool build_sdapi_img_gen_request(const json& j,
     }
 
     if (j.contains("lora") && j["lora"].is_array()) {
+        refresh_lora_cache(runtime);
         request.gen_params.lora_map.clear();
         request.gen_params.high_noise_lora_map.clear();
 
@@ -193,15 +185,25 @@ static bool build_sdapi_img_gen_request(const json& j,
         }
     }
 
-    if (img2img) {
-        const int expected_width  = request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0;
-        const int expected_height = request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0;
+    if (j.contains("ref_image_args")) {
+        if (!j["ref_image_args"].is_string()) {
+            error_message = "ref_image_args must be a string";
+            return false;
+        }
+        request.gen_params.ref_image_args = j["ref_image_args"].get<std::string>();
+    }
+
+    if (j.contains("image_preprocess") && !request.gen_params.parse_image_preprocess_json(j["image_preprocess"].dump())) {
+        error_message = "invalid image_preprocess";
+        return false;
+    }
 
+    if (img2img) {
         if (j.contains("init_images") && j["init_images"].is_array() && !j["init_images"].empty()) {
             if (decode_base64_image(j["init_images"][0].get<std::string>(),
-                                    3,
-                                    expected_width,
-                                    expected_height,
+                                    0,
+                                    0,
+                                    0,
                                     request.gen_params.init_image)) {
                 const sd_image_t& image = request.gen_params.init_image.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);
@@ -211,8 +213,8 @@ static bool build_sdapi_img_gen_request(const json& j,
         if (j.contains("mask") && j["mask"].is_string()) {
             if (decode_base64_image(j["mask"].get<std::string>(),
                                     1,
-                                    expected_width,
-                                    expected_height,
+                                    0,
+                                    0,
                                     request.gen_params.mask_image)) {
                 const sd_image_t& image = request.gen_params.mask_image.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);
@@ -225,9 +227,7 @@ static bool build_sdapi_img_gen_request(const json& j,
                 }
             }
         } else {
-            const int resolved_width  = request.gen_params.get_resolved_width();
-            const int resolved_height = request.gen_params.get_resolved_height();
-            assign_solid_mask(request.gen_params.mask_image, resolved_width, resolved_height);
+            request.gen_params.mask_image.reset({0, 0, 1, nullptr});
         }
 
         float denoising_strength = j.value("denoising_strength", -1.f);
@@ -243,9 +243,8 @@ static bool build_sdapi_img_gen_request(const json& j,
             }
             SDImageOwner image_owner;
             if (decode_base64_image(extra_image.get<std::string>(),
-                                    3,
-                                    request.gen_params.width_and_height_are_set() ? request.gen_params.width : 0,
-                                    request.gen_params.width_and_height_are_set() ? request.gen_params.height : 0,
+                                    0,
+                                    0, 0,
                                     image_owner)) {
                 const sd_image_t& image = image_owner.get();
                 request.gen_params.set_width_and_height_if_unset(image.width, image.height);
@@ -305,8 +304,79 @@ static nlohmann::json prepare_info_field(const SDContextParams& ctx_params,
     return jsoninfo;
 }
 
+struct SDAPIPreviewState {
+    std::mutex mutex;
+    std::string b64;
+    std::string id_task;
+    std::string job_type;
+    int step          = 0;
+    int total         = 0;
+    bool active       = false;
+    int64_t job_count = 0;
+    int job_no        = 0;
+};
+
+static SDAPIPreviewState g_preview;
+
+static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) {
+    (void)is_noisy;
+    (void)data;
+    if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) {
+        return;
+    }
+
+    const sd_image_t& frame      = frames[0];
+    const sd_preview_info_t info = sd_get_preview_info();
+
+    auto image_bytes = encode_image_to_vector(EncodedImageFormat::JPEG,
+                                              frame.data,
+                                              frame.width,
+                                              frame.height,
+                                              frame.channel,
+                                              "",
+                                              80);
+    if (image_bytes.empty()) {
+        return;
+    }
+
+    std::string b64 = base64_encode(image_bytes);
+    std::lock_guard<std::mutex> lock(g_preview.mutex);
+    g_preview.b64    = std::move(b64);
+    g_preview.step   = static_cast<int>(std::min<int64_t>(step < 0 ? -static_cast<int64_t>(step) : step,
+                                                        std::max(0, info.total_steps)));
+    g_preview.total  = std::max(0, info.total_steps);
+    g_preview.job_no = std::max(0, info.sample_pass - 1);
+}
+
+struct SDAPIPreviewGuard {
+    SDAPIPreviewGuard(const std::string& id_task, const sd_img_gen_params_t& params, const std::string& job_type) {
+        std::lock_guard<std::mutex> lock(g_preview.mutex);
+        g_preview.id_task  = id_task;
+        g_preview.total    = 0;
+        g_preview.job_type = job_type;
+        g_preview.b64.clear();
+        g_preview.step   = 0;
+        g_preview.active = true;
+        // Each batch item has a base pass and, when enabled, a hires pass.
+        g_preview.job_count = static_cast<int64_t>(std::max(1, params.batch_count)) * (params.hires.enabled ? 2 : 1);
+        g_preview.job_no    = 0;
+        sd_set_preview_callback(preview_callback, PREVIEW_PROJ, 1, true, false, nullptr);
+    }
+
+    ~SDAPIPreviewGuard() {
+        sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr);
+        std::lock_guard<std::mutex> lock(g_preview.mutex);
+        g_preview.active = false;
+        g_preview.b64.clear();
+    }
+
+    SDAPIPreviewGuard(const SDAPIPreviewGuard&)            = delete;
+    SDAPIPreviewGuard& operator=(const SDAPIPreviewGuard&) = delete;
+};
+
 void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
     ServerRuntime* runtime = &rt;
+    RuntimeRoutes routes(svr, rt);
 
     auto sdapi_any2img = [runtime](const httplib::Request& req, httplib::Response& res, bool img2img) {
         try {
@@ -321,7 +391,9 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 return;
             }
 
-            json j = json::parse(req.body);
+            std::string id_task = "sdapi";
+            json j              = json::parse(req.body);
+            id_task             = j.value("id_task", id_task);
             ImgGenJobRequest request;
             std::string error_message;
             if (!build_sdapi_img_gen_request(j, *runtime, img2img, request, error_message)) {
@@ -330,7 +402,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
                 return;
             }
 
-            LOG_VERBOSE("%s\n", request.gen_params.to_string().c_str());
+            LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
 
             sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
             SDImageVec results;
@@ -338,6 +410,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
 
             {
                 std::lock_guard<std::mutex> lock(*runtime->sd_ctx_mutex);
+                SDAPIPreviewGuard preview_guard(id_task, img_gen_params, img2img ? "img2img" : "txt2img");
                 sd_image_t* raw_results = nullptr;
                 if (!generate_image(runtime->sd_ctx, &img_gen_params, &raw_results, &num_results)) {
                     raw_results = nullptr;
@@ -404,15 +477,15 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         }
     };
 
-    svr.Post("/sdapi/v1/txt2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
+    routes.post("/sdapi/v1/txt2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
         sdapi_any2img(req, res, false);
     });
 
-    svr.Post("/sdapi/v1/img2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
+    routes.post("/sdapi/v1/img2img", [sdapi_any2img](const httplib::Request& req, httplib::Response& res) {
         sdapi_any2img(req, res, true);
     });
 
-    svr.Get("/sdapi/v1/loras", [runtime](const httplib::Request&, httplib::Response& res) {
+    routes.get("/sdapi/v1/loras", [runtime](const httplib::Request&, httplib::Response& res) {
         refresh_lora_cache(*runtime);
 
         json result = json::array();
@@ -420,8 +493,9 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             std::lock_guard<std::mutex> lock(*runtime->lora_mutex);
             for (const auto& e : *runtime->lora_cache) {
                 json item;
-                item["name"] = e.name;
-                item["path"] = e.path;
+                item["name"]         = e.name;
+                item["path"]         = e.path;
+                item["display_path"] = e.display_path;
                 result.push_back(item);
             }
         }
@@ -429,7 +503,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         res.set_content(result.dump(), "application/json");
     });
 
-    svr.Get("/sdapi/v1/upscalers", [runtime](const httplib::Request&, httplib::Response& res) {
+    routes.get("/sdapi/v1/upscalers", [runtime](const httplib::Request&, httplib::Response& res) {
         refresh_upscaler_cache(*runtime);
 
         auto make_builtin = [](const char* name) {
@@ -463,7 +537,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         res.set_content(result.dump(), "application/json");
     });
 
-    svr.Get("/sdapi/v1/latent-upscale-modes", [](const httplib::Request&, httplib::Response& res) {
+    routes.get("/sdapi/v1/latent-upscale-modes", [](const httplib::Request&, httplib::Response& res) {
         json result = json::array({
             {{"name", "Latent"}},
             {{"name", "Latent (nearest)"}},
@@ -475,7 +549,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         res.set_content(result.dump(), "application/json");
     });
 
-    svr.Get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) {
+    routes.get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) {
         std::vector<std::string> sampler_names;
         sampler_names.push_back("default");
         for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) {
@@ -492,7 +566,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         res.set_content(r.dump(), "application/json");
     });
 
-    svr.Get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) {
+    routes.get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) {
         std::vector<std::string> scheduler_names;
         scheduler_names.push_back("default");
         for (int i = 0; i < SCHEDULER_COUNT; i++) {
@@ -511,25 +585,85 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         res.set_content(r.dump(), "application/json");
     });
 
-    svr.Get("/sdapi/v1/sd-models", [runtime](const httplib::Request&, httplib::Response& res) {
+    routes.get("/sdapi/v1/sd-models", [runtime](const httplib::Request&, httplib::Response& res) {
         fs::path model_path = resolve_display_model_path(*runtime);
         json entry;
-        entry["title"]      = model_path.stem();
-        entry["model_name"] = model_path.stem();
-        entry["filename"]   = model_path.filename();
+        entry["title"]      = model_path.stem().u8string();
+        entry["model_name"] = model_path.stem().u8string();
+        entry["filename"]   = model_path.filename().u8string();
         entry["hash"]       = "8888888888";
         entry["sha256"]     = "8888888888888888888888888888888888888888888888888888888888888888";
         entry["config"]     = nullptr;
         json r              = json::array();
-        r.push_back(entry);
+        if (runtime->sd_ctx) {
+            r.push_back(entry);
+        }
         res.set_content(r.dump(), "application/json");
     });
 
-    svr.Get("/sdapi/v1/options", [runtime](const httplib::Request&, httplib::Response& res) {
+    routes.get("/sdapi/v1/options", [runtime](const httplib::Request&, httplib::Response& res) {
         fs::path model_path = resolve_display_model_path(*runtime);
         json r;
         r["samples_format"]      = "png";
-        r["sd_model_checkpoint"] = model_path.stem();
+        r["sd_model_checkpoint"] = model_path.stem().u8string();
+        res.set_content(r.dump(), "application/json");
+    });
+
+    routes.get("/sdapi/v1/progress", [](const httplib::Request& req, httplib::Response& res) {
+        bool skip_current_image = false;
+        bool has_id_param       = req.has_param("id_task");
+        std::string req_id_task = has_id_param ? req.get_param_value("id_task") : "";
+
+        if (req.has_param("skip_current_image")) {
+            std::string skip_str = req.get_param_value("skip_current_image");
+            skip_current_image   = (skip_str == "true" || skip_str == "1");
+        }
+
+        json r;
+        r["eta_relative"] = 0.0f;
+        json state;
+        state["skipped"]       = false;
+        state["interrupted"]   = false;
+        state["job_timestamp"] = "0";
+        r["textinfo"]          = nullptr;
+
+        {
+            std::lock_guard<std::mutex> lock(g_preview.mutex);
+
+            bool is_matching_task = g_preview.active && (!has_id_param || req_id_task == g_preview.id_task);
+
+            int step             = is_matching_task ? g_preview.step : 0;
+            int steps            = is_matching_task ? g_preview.total : 0;
+            std::string b64      = is_matching_task ? g_preview.b64 : "";
+            std::string job_type = is_matching_task ? g_preview.job_type : "";
+            int64_t job_count    = is_matching_task ? g_preview.job_count : 0;
+            int job_no           = is_matching_task ? g_preview.job_no : 0;
+
+            float batch_progress = (steps > 0) ? (static_cast<float>(step) / static_cast<float>(steps)) : 0.0f;
+            if (batch_progress > 1.0f)
+                batch_progress = 1.0f;
+
+            float overall_progress = job_count > 0
+                                         ? (static_cast<float>(job_no) + batch_progress) / static_cast<float>(job_count)
+                                         : 0.0f;
+            if (overall_progress > 1.0f)
+                overall_progress = 1.0f;
+
+            r["progress"]           = overall_progress;
+            state["job"]            = job_type;
+            state["sampling_step"]  = step;
+            state["sampling_steps"] = steps;
+            state["job_count"]      = job_count;
+            state["job_no"]         = job_no;
+            r["state"]              = state;
+
+            if (!skip_current_image && !b64.empty()) {
+                r["current_image"] = b64;
+            } else {
+                r["current_image"] = nullptr;
+            }
+        }
+
         res.set_content(r.dump(), "application/json");
     });
 }
diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp
index 7253dbf496..7e93e5edbf 100644
--- a/examples/server/routes_sdcpp.cpp
+++ b/examples/server/routes_sdcpp.cpp
@@ -3,12 +3,34 @@
 #include <algorithm>
 #include <cmath>
 #include <filesystem>
+#include <limits>
 
 #include "async_jobs.h"
 #include "common/common.h"
+#include "common/media_io.h"
+#include "common/resource_owners.hpp"
+#include "model_catalog.h"
 
 namespace fs = std::filesystem;
 
+static constexpr uint32_t k_max_upscale_dimension = 8192;
+
+static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int repeats) {
+    if (image.width == 0 || image.height == 0 || factor < 1 || repeats < 1 || repeats > 4) {
+        return false;
+    }
+    uint32_t width  = image.width;
+    uint32_t height = image.height;
+    for (int i = 0; i < repeats; ++i) {
+        if (width > k_max_upscale_dimension / factor || height > k_max_upscale_dimension / factor) {
+            return false;
+        }
+        width *= factor;
+        height *= factor;
+    }
+    return true;
+}
+
 static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) {
     if (mode_str == "disabled") {
         mode_out = SD_CACHE_DISABLED;
@@ -57,11 +79,11 @@ static json make_vae_tiling_json(const sd_tiling_params_t& params) {
     return {
         {"enabled", params.enabled},
         {"temporal_tiling", params.temporal_tiling},
-        {"tile_size_x", params.tile_size_x},
-        {"tile_size_y", params.tile_size_y},
+        {"tile_size_w", params.tile_size_w},
+        {"tile_size_h", params.tile_size_h},
         {"target_overlap", params.target_overlap},
-        {"rel_size_x", params.rel_size_x},
-        {"rel_size_y", params.rel_size_y},
+        {"rel_size_w", params.rel_size_w},
+        {"rel_size_h", params.rel_size_h},
         {"extra_tiling_args", params.extra_tiling_args ? params.extra_tiling_args : ""},
     };
 }
@@ -69,20 +91,21 @@ static json make_vae_tiling_json(const sd_tiling_params_t& params) {
 static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
     const auto& ctx = *runtime.ctx_params;
     if (!ctx.model_path.empty()) {
-        return fs::path(ctx.model_path);
+        return fs::u8path(ctx.model_path);
     }
     if (!ctx.diffusion_model_path.empty()) {
-        return fs::path(ctx.diffusion_model_path);
+        return fs::u8path(ctx.diffusion_model_path);
     }
     return {};
 }
 
-static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector<int>& skip_layers) {
+static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector<int>& skip_layers, const std::vector<float>& custom_sigmas = {}) {
     const auto& guidance = sample_params.guidance;
     return {
         {"scheduler", capability_scheduler_name(sample_params.scheduler)},
         {"sample_method", capability_sample_method_name(sample_params.sample_method)},
         {"sample_steps", sample_params.sample_steps},
+        {"custom_sigmas", custom_sigmas},
         {"eta", finite_number_or_null(sample_params.eta)},
         {"shifted_timestep", sample_params.shifted_timestep},
         {"flow_shift", finite_number_or_null(sample_params.flow_shift)},
@@ -127,11 +150,12 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const
         {"seed", defaults.seed},
         {"batch_count", defaults.batch_count},
         {"qwen_image_layers", defaults.qwen_image_layers},
-        {"auto_resize_ref_image", defaults.auto_resize_ref_image},
+        {"ref_image_args", defaults.ref_image_args},
+        {"image_preprocess", defaults.image_preprocess},
         {"increase_ref_index", defaults.increase_ref_index},
         {"control_strength", defaults.control_strength},
         {"ip_adapter_strength", defaults.ip_adapter_strength},
-        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
+        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers, defaults.custom_sigmas)},
         {"hires", make_hires_json(defaults)},
         {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
         {"cache_mode", defaults.cache_mode},
@@ -153,10 +177,11 @@ static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const
         {"strength", defaults.strength},
         {"seed", defaults.seed},
         {"video_frames", defaults.video_frames},
+        {"image_preprocess", defaults.image_preprocess},
         {"fps", defaults.fps},
         {"moe_boundary", defaults.moe_boundary},
         {"vace_strength", defaults.vace_strength},
-        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers)},
+        {"sample_params", make_sample_params_json(defaults.sample_params, defaults.skip_layers, defaults.custom_sigmas)},
         {"high_noise_sample_params", make_sample_params_json(defaults.high_noise_sample_params, defaults.high_noise_skip_layers)},
         {"hires", make_hires_json(defaults)},
         {"vae_tiling_params", make_vae_tiling_json(defaults.vae_tiling_params)},
@@ -182,6 +207,7 @@ static json make_img_gen_features_json() {
         {"cache", true},
         {"cancel_queued", true},
         {"cancel_generating", false},
+        {"preview", true},
     };
 }
 
@@ -196,9 +222,18 @@ static json make_vid_gen_features_json() {
         {"cache", true},
         {"cancel_queued", true},
         {"cancel_generating", false},
+        {"preview", true},
     };
 }
 
+static json make_preview_modes_json() {
+    json modes = json::array();
+    for (int i = 0; i < PREVIEW_COUNT; ++i) {
+        modes.push_back(sd_preview_name((preview_t)i));
+    }
+    return modes;
+}
+
 static json make_capabilities_json(ServerRuntime& runtime) {
     refresh_lora_cache(runtime);
     refresh_upscaler_cache(runtime);
@@ -233,43 +268,66 @@ static json make_capabilities_json(ServerRuntime& runtime) {
             available_loras.push_back({
                 {"name", entry.name},
                 {"path", entry.path},
+                {"display_path", entry.display_path},
             });
         }
     }
 
     available_upscalers.push_back({
         {"name", "None"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Lanczos"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Nearest"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (nearest)"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (nearest-exact)"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (antialiased)"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (bicubic)"},
+        {"model", false},
+        {"image_upscale", false},
     });
     available_upscalers.push_back({
         {"name", "Latent (bicubic antialiased)"},
+        {"model", false},
+        {"image_upscale", false},
     });
+    bool have_upscaler_models = false;
     {
         std::lock_guard<std::mutex> lock(*runtime.upscaler_mutex);
         for (const auto& entry : *runtime.upscaler_cache) {
             available_upscalers.push_back({
                 {"name", entry.name},
+                {"model", true},
+                {"image_upscale", entry.image_upscale_factor > 0},
             });
+            have_upscaler_models = have_upscaler_models || entry.image_upscale_factor > 0;
         }
     }
 
@@ -289,18 +347,34 @@ static json make_capabilities_json(ServerRuntime& runtime) {
         default_vid_output_format = video_output_formats[0].get<std::string>();
     }
 
+    const json generation_features = model_generation_features(runtime);
+    auto recipe_features           = [&](json features, bool video) {
+        features["init_image"] = generation_features.at(video ? "init_image" : "img2img");
+        for (const char* key : {"mask_image", "ref_images", "end_image", "control_frames", "lora", "vae_tiling", "cache", "preview"}) {
+            if (features.contains(key) || std::string(key) == "ref_images")
+                features[key] = generation_features.at(key);
+        }
+        if (video)
+            features["high_noise_sample_params"] = generation_features.at("high_noise");
+        else {
+            features["mask_image"]       = generation_features.at("mask_image").get<bool>() && generation_features.at("img2img").get<bool>();
+            features["control_image"]    = generation_features.at("controlnet").get<bool>() || generation_features.at("control_image").get<bool>();
+            features["ip_adapter_image"] = generation_features.at("ip_adapter");
+        }
+        return features;
+    };
     json defaults_by_mode       = json::object();
     json output_formats_by_mode = json::object();
     json features_by_mode       = json::object();
     if (supports_img) {
         defaults_by_mode["img_gen"]       = make_img_gen_defaults_json(defaults, default_img_output_format);
         output_formats_by_mode["img_gen"] = image_output_formats;
-        features_by_mode["img_gen"]       = make_img_gen_features_json();
+        features_by_mode["img_gen"]       = recipe_features(make_img_gen_features_json(), false);
     }
     if (supports_vid) {
         defaults_by_mode["vid_gen"]       = make_vid_gen_defaults_json(defaults, default_vid_output_format);
         output_formats_by_mode["vid_gen"] = video_output_formats;
-        features_by_mode["vid_gen"]       = make_vid_gen_features_json();
+        features_by_mode["vid_gen"]       = recipe_features(make_vid_gen_features_json(), true);
     }
 
     json top_level_defaults       = json::object();
@@ -328,17 +402,21 @@ static json make_capabilities_json(ServerRuntime& runtime) {
         {"stem", model_path.stem().u8string()},
         {"path", model_path.u8string()},
     };
-    result["current_mode"]     = current_mode;
-    result["supported_modes"]  = supported_modes;
-    result["defaults"]         = top_level_defaults;
-    result["defaults_by_mode"] = defaults_by_mode;
-    result["limits"]           = {
-                  {"min_width", 64},
-                  {"max_width", 4096},
-                  {"min_height", 64},
-                  {"max_height", 4096},
-                  {"max_batch_count", 8},
-                  {"max_queue_size", manager.max_pending_jobs},
+    result["generation_features"] = generation_features;
+    result["runtime"]             = model_status(runtime);
+    result["current_mode"]        = current_mode;
+    result["supported_modes"]     = supported_modes;
+    result["defaults"]            = top_level_defaults;
+    result["defaults_by_mode"]    = defaults_by_mode;
+    result["limits"]              = {
+                     {"min_width", 64},
+                     {"max_width", 4096},
+                     {"min_height", 64},
+                     {"max_height", 4096},
+                     {"max_batch_count", 8},
+                     {"max_queue_size", manager.max_pending_jobs},
+                     {"max_upscale_width", k_max_upscale_dimension},
+                     {"max_upscale_height", k_max_upscale_dimension},
     };
     result["samplers"]               = samplers;
     result["schedulers"]             = schedulers;
@@ -348,6 +426,8 @@ static json make_capabilities_json(ServerRuntime& runtime) {
     result["features_by_mode"]       = features_by_mode;
     result["loras"]                  = available_loras;
     result["upscalers"]              = available_upscalers;
+    result["upscale"]                = have_upscaler_models;
+    result["preview_modes"]          = make_preview_modes_json();
     return result;
 }
 
@@ -370,6 +450,8 @@ static bool parse_img_gen_request(const json& body,
     if (!assign_output_options(request, output_format, output_compression, true, error_message)) {
         return false;
     }
+    request.preview_mode     = body.value("preview", std::string("none"));
+    request.preview_interval = body.value("preview_interval", 1);
     // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
     if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
         error_message = "invalid generation parameters";
@@ -397,6 +479,8 @@ static bool parse_vid_gen_request(const json& body,
     if (!assign_output_options(request, output_format, output_compression, error_message)) {
         return false;
     }
+    request.preview_mode     = body.value("preview", std::string("none"));
+    request.preview_interval = body.value("preview_interval", 1);
     // Intentionally disable prompt-embedded LoRA tag parsing for server APIs.
     if (!request.gen_params.resolve_and_validate(VID_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) {
         error_message = "invalid generation parameters";
@@ -407,13 +491,179 @@ static bool parse_vid_gen_request(const json& body,
 
 void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
     ServerRuntime* runtime = &rt;
+    RuntimeRoutes routes(svr, rt);
 
-    svr.Get("/sdcpp/v1/capabilities", [runtime](const httplib::Request&, httplib::Response& res) {
+    routes.get("/sdcpp/v1/capabilities", [runtime](const httplib::Request&, httplib::Response& res) {
         res.status = 200;
         res.set_content(make_capabilities_json(*runtime).dump(), "application/json");
     });
 
-    svr.Post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
+    routes.post("/sdcpp/v1/upscale", [runtime](const httplib::Request& req, httplib::Response& res) {
+        try {
+            if (req.body.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"empty body"})", "application/json");
+                return;
+            }
+            json body = json::parse(req.body);
+            if (!body.is_object()) {
+                res.status = 400;
+                res.set_content(R"({"error":"body must be an object"})", "application/json");
+                return;
+            }
+            for (const char* key : {"repeats", "tile_size", "output_compression"}) {
+                if (!body.contains(key)) {
+                    continue;
+                }
+                const auto& value = body[key];
+                const bool valid  = value.is_number_unsigned()
+                                        ? value.get<uint64_t>() <= static_cast<uint64_t>(std::numeric_limits<int>::max())
+                                        : value.is_number_integer() && value.get<int64_t>() >= std::numeric_limits<int>::min() &&
+                                             value.get<int64_t>() <= std::numeric_limits<int>::max();
+                if (!valid) {
+                    res.status = 400;
+                    res.set_content(json({{"error", std::string(key) + " must be a 32-bit integer"}}).dump(), "application/json");
+                    return;
+                }
+            }
+            ImgGenJobRequest output_options;
+            std::string error_message;
+            if (!assign_output_options(output_options,
+                                       body.value("output_format", std::string("png")),
+                                       body.value("output_compression", 100),
+                                       true,
+                                       error_message)) {
+                res.status = 400;
+                res.set_content(json({{"error", error_message}}).dump(), "application/json");
+                return;
+            }
+            const int tile_size      = std::max(32, body.value("tile_size", runtime->default_gen_params->upscale_tile_size));
+            const int repeats        = std::clamp(body.value("repeats", 1), 1, 4);
+            const std::string wanted = body.value("upscaler", std::string());
+
+            const std::string encoded = body.value("image", std::string());
+            if (encoded.empty()) {
+                res.status = 400;
+                res.set_content(R"({"error":"image is required"})", "application/json");
+                return;
+            }
+            SDImageOwner input;
+            if (!decode_base64_image(encoded, 3, 0, 0, input) || input.get().data == nullptr) {
+                res.status = 400;
+                res.set_content(R"({"error":"image could not be read"})", "application/json");
+                return;
+            }
+
+            refresh_upscaler_cache(*runtime);
+            int model_scale = 0;
+            std::string model_path;
+            std::string used_name;
+            {
+                std::lock_guard<std::mutex> lock(*runtime->upscaler_mutex);
+                for (const auto& entry : *runtime->upscaler_cache) {
+                    if (entry.image_upscale_factor > 0 && (wanted.empty() || entry.name == wanted)) {
+                        model_path  = entry.fullpath;
+                        used_name   = entry.name;
+                        model_scale = entry.image_upscale_factor;
+                        break;
+                    }
+                }
+            }
+            if (model_path.empty()) {
+                res.status = 400;
+                res.set_content(json({{"error", wanted.empty()
+                                                    ? std::string("no RGB ESRGAN upscaler models are available; "
+                                                                  "start the server with --hires-upscalers-dir")
+                                                    : "no compatible image upscaler called " + wanted}})
+                                    .dump(),
+                                "application/json");
+                return;
+            }
+
+            if (!valid_upscale_dimensions(input.get(), model_scale, repeats)) {
+                res.status = 400;
+                res.set_content(R"({"error":"upscaled dimensions must not exceed 8192 x 8192"})", "application/json");
+                return;
+            }
+
+            // One GPU: an upscale must not run while a generation is using it.
+            std::lock_guard<std::mutex> ctx_lock(*runtime->sd_ctx_mutex);
+            UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(model_path.c_str(),
+                                                         runtime->ctx_params->diffusion_conv_direct,
+                                                         runtime->ctx_params->n_threads,
+                                                         tile_size,
+                                                         runtime->ctx_params->backend.c_str(),
+                                                         runtime->ctx_params->params_backend.c_str()));
+            if (upscaler_ctx == nullptr) {
+                res.status = 500;
+                res.set_content(R"({"error":"the upscaler model could not be loaded"})", "application/json");
+                return;
+            }
+            const int factor = get_upscale_factor(upscaler_ctx.get());
+            // The model file may have changed since its metadata was cached.
+            if (!valid_upscale_dimensions(input.get(), factor, repeats)) {
+                res.status = 400;
+                res.set_content(R"({"error":"upscaled dimensions must not exceed 8192 x 8192"})", "application/json");
+                return;
+            }
+
+            SDImageOwner current(input.release());
+            for (int i = 0; i < repeats; ++i) {
+                sd_image_t* out_images = nullptr;
+                int out_count          = 0;
+                if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &out_images, &out_count) ||
+                    out_count <= 0 || out_images[0].data == nullptr) {
+                    free_sd_images(out_images, out_count);
+                    res.status = 500;
+                    res.set_content(R"({"error":"upscale failed"})", "application/json");
+                    return;
+                }
+                sd_image_t produced = out_images[0];
+                out_images[0]       = {0, 0, 0, nullptr};
+                free_sd_images(out_images, out_count);
+                current.reset(produced);
+            }
+
+            const std::string& format = output_options.output_format;
+            const int compression     = output_options.output_compression;
+            const sd_image_t result   = current.get();
+            auto image_bytes          = encode_image_to_vector(format == "jpeg"   ? EncodedImageFormat::JPEG
+                                                               : format == "webp" ? EncodedImageFormat::WEBP
+                                                                                  : EncodedImageFormat::PNG,
+                                                      result.data,
+                                                      result.width,
+                                                      result.height,
+                                                      result.channel,
+                                                      "",
+                                                      compression);
+            if (image_bytes.empty()) {
+                res.status = 500;
+                res.set_content(R"({"error":"the result could not be encoded"})", "application/json");
+                return;
+            }
+
+            json out;
+            out["upscaler"]      = used_name;
+            out["scale"]         = factor;
+            out["repeats"]       = repeats;
+            out["width"]         = result.width;
+            out["height"]        = result.height;
+            out["output_format"] = format;
+            json images          = json::array();
+            images.push_back({{"index", 0}, {"b64_json", base64_encode(image_bytes)}});
+            out["images"] = std::move(images);
+            res.set_content(out.dump(), "application/json");
+            res.status = 200;
+        } catch (const json::exception& e) {
+            res.status = 400;
+            res.set_content(json({{"error", "invalid request"}, {"message", e.what()}}).dump(), "application/json");
+        } catch (const std::exception& e) {
+            res.status = 500;
+            res.set_content(json({{"error", std::string("server_error: ") + e.what()}}).dump(), "application/json");
+        }
+    });
+
+    routes.post("/sdcpp/v1/img_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
         try {
             if (req.body.empty()) {
                 res.status = 400;
@@ -439,7 +689,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
             job->kind                               = AsyncJobKind::ImgGen;
             job->status                             = AsyncJobStatus::Queued;
-            job->created_at                         = unix_timestamp_now();
+            job->created_at                         = unix_timestamp_now_ms();
             job->img_gen                            = std::move(request);
 
             {
@@ -475,7 +725,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         }
     });
 
-    svr.Post("/sdcpp/v1/vid_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
+    routes.post("/sdcpp/v1/vid_gen", [runtime](const httplib::Request& req, httplib::Response& res) {
         try {
             if (req.body.empty()) {
                 res.status = 400;
@@ -501,7 +751,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
             std::shared_ptr<AsyncGenerationJob> job = std::make_shared<AsyncGenerationJob>();
             job->kind                               = AsyncJobKind::VidGen;
             job->status                             = AsyncJobStatus::Queued;
-            job->created_at                         = unix_timestamp_now();
+            job->created_at                         = unix_timestamp_now_ms();
             job->vid_gen                            = std::move(request);
 
             {
@@ -537,7 +787,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         }
     });
 
-    svr.Get(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+))", [runtime](const httplib::Request& req, httplib::Response& res) {
+    routes.get(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+))", [runtime](const httplib::Request& req, httplib::Response& res) {
         AsyncJobManager& manager = *runtime->async_job_manager;
         std::lock_guard<std::mutex> lock(manager.mutex);
         purge_expired_jobs(manager);
@@ -559,7 +809,7 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
         res.set_content(make_async_job_json(manager, *it->second).dump(), "application/json");
     });
 
-    svr.Post(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+)/cancel)", [runtime](const httplib::Request& req, httplib::Response& res) {
+    routes.post(R"(/sdcpp/v1/jobs/([A-Za-z0-9_\-]+)/cancel)", [runtime](const httplib::Request& req, httplib::Response& res) {
         AsyncJobManager& manager = *runtime->async_job_manager;
         std::lock_guard<std::mutex> lock(manager.mutex);
         purge_expired_jobs(manager);
diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp
index cd0e2e722b..34140ea353 100644
--- a/examples/server/runtime.cpp
+++ b/examples/server/runtime.cpp
@@ -11,6 +11,10 @@
 
 #include "common/common.h"
 #include "common/log.h"
+#include "model_catalog.h"
+
+#include <map>
+#include <set>
 
 namespace fs = std::filesystem;
 
@@ -22,7 +26,7 @@ static std::string lower_ascii(std::string value) {
 }
 
 static bool is_supported_model_ext(const fs::path& p) {
-    auto ext = lower_ascii(p.extension().string());
+    auto ext = lower_ascii(p.extension().u8string());
     return ext == ".gguf" || ext == ".pt" || ext == ".pth" || ext == ".safetensors";
 }
 
@@ -210,6 +214,13 @@ ArgOptions SDSvrParams::get_options() {
 
     options.manual_options = {
         {"-h", "--help", "show this help message and exit", on_help_arg},
+        {"", "--model-dir", "model root; repeat to search multiple directories (default: models)",
+         [this](int argc, const char** argv, int index) {
+             if (++index >= argc || argv[index][0] == '\0')
+                 return -1;
+             model_dirs.push_back(argv_to_utf8(index, argv));
+             return 1;
+         }},
     };
     add_log_options(options, log_level);
     return options;
@@ -226,7 +237,7 @@ bool SDSvrParams::validate() {
         return false;
     }
 
-    if (!serve_html_path.empty() && !fs::exists(serve_html_path)) {
+    if (!serve_html_path.empty() && !fs::exists(fs::u8path(serve_html_path))) {
         LOG_ERROR("error: serve_html_path file does not exist: %s", serve_html_path.c_str());
         return false;
     }
@@ -253,70 +264,155 @@ std::string SDSvrParams::to_string() const {
 
 void refresh_lora_cache(ServerRuntime& rt) {
     std::vector<LoraEntry> new_cache;
-
-    fs::path lora_dir = rt.ctx_params->lora_model_dir;
-    if (fs::exists(lora_dir) && fs::is_directory(lora_dir)) {
-        for (auto& entry : fs::recursive_directory_iterator(lora_dir, fs::directory_options::skip_permission_denied)) {
-            if (!entry.is_regular_file()) {
-                continue;
+    std::vector<ModelRoot> directories;
+    if (!rt.ctx_params->lora_model_dir.empty()) {
+        directories.push_back(make_model_root(fs::u8path(rt.ctx_params->lora_model_dir), true));
+    } else {
+        for (const auto& root : get_model_roots(*rt.svr_params)) {
+            try {
+                const auto paths = model_category_directories(root, "loras");
+                for (size_t i = 0; i < paths.size(); ++i) {
+                    if (!model_path_inside(root.path, fs::canonical(paths[i]))) {
+                        LOG_WARN("LoRA directory escapes model directory: %s", paths[i].u8string().c_str());
+                        continue;
+                    }
+                    directories.push_back(make_model_root(paths[i], root.primary && lower_ascii(paths[i].filename().u8string()) == "loras"));
+                }
+            } catch (const std::exception& e) {
+                LOG_WARN("error while finding LoRA directories in %s: %s", root.label.c_str(), e.what());
             }
-            const fs::path& p = entry.path();
-            if (!is_supported_model_ext(p)) {
+        }
+    }
+    std::set<std::string> seen;
+    std::map<std::string, size_t> names;
+    for (const auto& directory : directories) {
+        try {
+            if (!fs::exists(directory.path))
                 continue;
+            for (fs::recursive_directory_iterator it(directory.path), end; it != end; ++it) {
+                std::error_code ec;
+                const auto canonical = fs::canonical(it->path(), ec);
+                if (ec || !model_path_inside(directory.path, canonical)) {
+                    it.disable_recursion_pending();
+                    LOG_WARN("skipping inaccessible LoRA or path outside LoRA directory: %s", it->path().u8string().c_str());
+                    continue;
+                }
+                const auto ext = lower_ascii(it->path().extension().u8string());
+                if (!it->is_regular_file() || (!is_supported_model_ext(it->path()) && ext != ".ckpt" && ext != ".sft") ||
+                    !seen.insert(model_path_key(canonical)).second)
+                    continue;
+                const auto relative = it->path().lexically_relative(directory.path).generic_u8string();
+                LoraEntry entry;
+                entry.name         = it->path().stem().u8string();
+                entry.path         = directory.primary ? relative : directory.id + "/" + relative;
+                entry.fullpath     = it->path().u8string();
+                entry.display_path = relative;
+                entry.root         = directory.path;
+                ++names[relative];
+                new_cache.push_back(std::move(entry));
             }
-
-            LoraEntry lora_entry;
-            lora_entry.name     = p.stem().u8string();
-            lora_entry.fullpath = p.u8string();
-            std::string rel     = p.lexically_relative(lora_dir).u8string();
-            std::replace(rel.begin(), rel.end(), '\\', '/');
-            lora_entry.path = rel;
-
-            new_cache.push_back(std::move(lora_entry));
+        } catch (const std::exception& e) {
+            LOG_WARN("error while scanning LoRA directory '%s': %s", directory.path.u8string().c_str(), e.what());
         }
     }
-
+    for (auto& entry : new_cache) {
+        if (names[entry.display_path] > 1)
+            entry.display_path = fs::u8path(entry.fullpath).generic_u8string();
+    }
     std::sort(new_cache.begin(), new_cache.end(), [](const LoraEntry& a, const LoraEntry& b) {
         return a.path < b.path;
     });
-
-    {
-        std::lock_guard<std::mutex> lock(*rt.lora_mutex);
-        *rt.lora_cache = std::move(new_cache);
-    }
+    std::lock_guard<std::mutex> lock(*rt.lora_mutex);
+    *rt.lora_cache = std::move(new_cache);
 }
 
 std::string get_lora_full_path(ServerRuntime& rt, const std::string& path) {
     std::lock_guard<std::mutex> lock(*rt.lora_mutex);
     auto it = std::find_if(rt.lora_cache->begin(), rt.lora_cache->end(),
                            [&](const LoraEntry& entry) { return entry.path == path; });
-    return it != rt.lora_cache->end() ? it->fullpath : "";
+    if (it == rt.lora_cache->end())
+        return "";
+    std::error_code ec;
+    const auto canonical = fs::canonical(fs::u8path(it->fullpath), ec);
+    if (ec || !model_path_inside(it->root, canonical) || !fs::is_regular_file(canonical, ec) || ec)
+        return "";
+    return canonical.u8string();
 }
 
 void refresh_upscaler_cache(ServerRuntime& rt) {
     std::vector<UpscalerEntry> new_cache;
+    std::vector<UpscalerEntry> previous_cache;
+    {
+        std::lock_guard<std::mutex> lock(*rt.upscaler_mutex);
+        previous_cache = *rt.upscaler_cache;
+    }
 
-    fs::path upscaler_dir = rt.ctx_params->hires_upscalers_dir;
-    if (fs::exists(upscaler_dir) && fs::is_directory(upscaler_dir)) {
-        for (auto& entry : fs::directory_iterator(upscaler_dir)) {
-            if (!entry.is_regular_file()) {
-                continue;
+    fs::path upscaler_dir = fs::u8path(rt.ctx_params->hires_upscalers_dir);
+    std::error_code ec;
+    if (fs::exists(upscaler_dir, ec) && !ec && fs::is_directory(upscaler_dir, ec) && !ec) {
+        try {
+            auto it = fs::directory_iterator(
+                upscaler_dir,
+                fs::directory_options::skip_permission_denied,
+                ec);
+            auto end = fs::directory_iterator();
+            while (!ec && it != end) {
+                std::error_code entry_ec;
+                bool is_reg = it->is_regular_file(entry_ec);
+                if (entry_ec || !is_reg) {
+                    it.increment(ec);
+                    continue;
+                }
+                const fs::path& p = it->path();
+                if (!is_supported_model_ext(p)) {
+                    it.increment(ec);
+                    continue;
+                }
+
+                UpscalerEntry upscaler_entry;
+                upscaler_entry.name = p.stem().u8string();
+                std::error_code abs_ec;
+                fs::path abs_path         = fs::absolute(p, abs_ec);
+                upscaler_entry.fullpath   = (abs_ec ? p : abs_path.lexically_normal()).u8string();
+                upscaler_entry.model_name = "ESRGAN_4x";
+                upscaler_entry.path       = p.filename().u8string();
+
+                std::error_code size_ec;
+                upscaler_entry.file_size = it->file_size(size_ec);
+                if (size_ec) {
+                    it.increment(ec);
+                    continue;
+                }
+
+                std::error_code time_ec;
+                upscaler_entry.last_modified        = it->last_write_time(time_ec);
+                auto previous                       = std::find_if(previous_cache.begin(), previous_cache.end(), [&](const UpscalerEntry& cached) {
+                    return cached.fullpath == upscaler_entry.fullpath &&
+                           cached.file_size == upscaler_entry.file_size &&
+                           (!time_ec && cached.last_modified == upscaler_entry.last_modified);
+                });
+                upscaler_entry.image_upscale_factor = previous != previous_cache.end()
+                                                          ? previous->image_upscale_factor
+                                                          : get_upscaler_model_scale(upscaler_entry.fullpath.c_str());
+                if (upscaler_entry.image_upscale_factor > 0) {
+                    upscaler_entry.scale      = upscaler_entry.image_upscale_factor;
+                    upscaler_entry.model_name = "ESRGAN_" + std::to_string(upscaler_entry.scale) + "x";
+                }
+
+                new_cache.push_back(std::move(upscaler_entry));
+                it.increment(ec);
             }
-            const fs::path& p = entry.path();
-            if (!is_supported_model_ext(p)) {
-                continue;
-            }
-
-            UpscalerEntry upscaler_entry;
-            upscaler_entry.name       = p.stem().u8string();
-            upscaler_entry.fullpath   = fs::absolute(p).lexically_normal().u8string();
-            upscaler_entry.model_name = "ESRGAN_4x";
-            upscaler_entry.path       = p.filename().u8string();
-
-            new_cache.push_back(std::move(upscaler_entry));
+        } catch (const std::exception& e) {
+            LOG_WARN("error while scanning upscalers directory '%s': %s", upscaler_dir.u8string().c_str(), e.what());
+            return;
         }
     }
 
+    if (ec) {
+        LOG_WARN("error while scanning upscalers directory '%s': %s", upscaler_dir.u8string().c_str(), ec.message().c_str());
+        return;
+    }
+
     std::sort(new_cache.begin(), new_cache.end(), [](const UpscalerEntry& a, const UpscalerEntry& b) {
         return a.name < b.name;
     });
@@ -327,8 +423,8 @@ void refresh_upscaler_cache(ServerRuntime& rt) {
     }
 }
 
-int64_t unix_timestamp_now() {
-    return std::chrono::duration_cast<std::chrono::seconds>(
+int64_t unix_timestamp_now_ms() {
+    return std::chrono::duration_cast<std::chrono::milliseconds>(
                std::chrono::system_clock::now().time_since_epoch())
         .count();
 }
diff --git a/examples/server/runtime.h b/examples/server/runtime.h
index 677bb0ba0c..d9272f6c24 100644
--- a/examples/server/runtime.h
+++ b/examples/server/runtime.h
@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <cstdint>
+#include <filesystem>
 #include <mutex>
 #include <string>
 #include <vector>
@@ -16,14 +17,16 @@ using json = nlohmann::json;
 struct ArgOptions;
 struct SDContextParams;
 struct AsyncJobManager;
+struct ServerModels;
 
 struct SDSvrParams {
     std::string listen_ip = "127.0.0.1";
     int listen_port       = 1234;
     std::string serve_html_path;
-    bool normal_exit = false;
+    std::vector<std::string> model_dirs;
+    bool normal_exit         = false;
     sd_log_level_t log_level = SD_LOG_INFO;
-    bool color       = false;
+    bool color               = false;
 
     ArgOptions get_options();
     bool validate();
@@ -35,6 +38,8 @@ struct LoraEntry {
     std::string name;
     std::string path;
     std::string fullpath;
+    std::string display_path;
+    std::filesystem::path root;
 };
 
 struct UpscalerEntry {
@@ -42,26 +47,32 @@ struct UpscalerEntry {
     std::string path;
     std::string fullpath;
     std::string model_name;
-    int scale = 4;
+    int scale                = 4;
+    int image_upscale_factor = 0;
+    uintmax_t file_size      = 0;
+    std::filesystem::file_time_type last_modified;
 };
 
 struct ServerRuntime {
     sd_ctx_t* sd_ctx;
     std::mutex* sd_ctx_mutex;
     const SDSvrParams* svr_params;
-    const SDContextParams* ctx_params;
-    const SDGenerationParams* default_gen_params;
+    SDContextParams* ctx_params;
+    SDGenerationParams* default_gen_params;
     std::vector<LoraEntry>* lora_cache;
     std::mutex* lora_mutex;
     std::vector<UpscalerEntry>* upscaler_cache;
     std::mutex* upscaler_mutex;
     AsyncJobManager* async_job_manager;
+    ServerModels* models;
 };
 
 struct ImgGenJobRequest {
     SDGenerationParams gen_params;
     std::string output_format = "png";
     int output_compression    = 100;
+    std::string preview_mode  = "none";
+    int preview_interval      = 1;
 
     sd_img_gen_params_t to_sd_img_gen_params_t() {
         return gen_params.to_sd_img_gen_params_t();
@@ -72,6 +83,8 @@ struct VidGenJobRequest {
     SDGenerationParams gen_params;
     std::string output_format = "webm";
     int output_compression    = 100;
+    std::string preview_mode  = "none";
+    int preview_interval      = 1;
 
     sd_vid_gen_params_t to_sd_vid_gen_params_t() {
         return gen_params.to_sd_vid_gen_params_t();
@@ -97,4 +110,4 @@ std::string unsupported_generation_mode_error(SDMode mode);
 void refresh_lora_cache(ServerRuntime& rt);
 std::string get_lora_full_path(ServerRuntime& rt, const std::string& path);
 void refresh_upscaler_cache(ServerRuntime& rt);
-int64_t unix_timestamp_now();
+int64_t unix_timestamp_now_ms();
diff --git a/ggml b/ggml
index f583f393cd..dc998771e7 160000
--- a/ggml
+++ b/ggml
@@ -1 +1 @@
-Subproject commit f583f393cd5dfdc129360bbf75cb3d49ccc837a8
+Subproject commit dc998771e700d49bbdf9a39cede22bb5b3311767
diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h
index 9173240be6..404d8cbddc 100644
--- a/include/stable-diffusion.h
+++ b/include/stable-diffusion.h
@@ -173,11 +173,13 @@ enum lora_apply_mode_t {
 typedef struct {
     bool enabled;
     bool temporal_tiling;
-    int tile_size_x;
-    int tile_size_y;
+    // Spatial tile dimensions in image pixels for both encode and decode; 0 uses 256.
+    int tile_size_w;
+    int tile_size_h;
     float target_overlap;
-    float rel_size_x;
-    float rel_size_y;
+    // Positive values override tile_size: <= 1 is a dimension fraction, > 1 a target tile count.
+    float rel_size_w;
+    float rel_size_h;
     const char* extra_tiling_args;
 } sd_tiling_params_t;
 
@@ -243,10 +245,14 @@ typedef struct {
     const char* rpc_servers;
     const char* model_args;
     bool disable_segmented_compute;  // Force monolithic graph execution even when automatic graph cutting would fit memory better
+    bool batched_cfg;                // Run the conditional and unconditional CFG branches in one batched UNet forward when supported
     float linear_scale;              // Override linear input scaling; 0 keeps the model default
     float attn_scale;                // Override flash-attention K/V scaling; 0 keeps the model default
     const char* tokenizer;           // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens
     bool sage_attn;
+    int conditioning_cache_size;  // Maximum cached conditioning entries per context; 0 disables caching (default: 4)
+    const char* flash_attn_mode;  // on/off/auto or module assignments (e.g. "auto,te=off"); NULL/empty honors legacy flags.
+    const char* conv_direct_mode;  // on/off/auto or diffusion/vae/controlnet assignments; NULL/empty honors legacy flags.
 } sd_ctx_params_t;
 
 typedef struct {
@@ -263,6 +269,11 @@ typedef struct {
     uint8_t* data;
 } sd_image_t;
 
+typedef struct {
+    // Semicolon-separated target=...,key=value rules. NULL preserves defaults.
+    const char* rules;
+} sd_image_preprocess_params_t;
+
 typedef struct {
     sd_image_t* frames;
     int frame_count;
@@ -410,6 +421,7 @@ typedef struct {
     int qwen_image_layers;
     bool circular_x;
     bool circular_y;
+    sd_image_preprocess_params_t image_preprocess;
 } sd_img_gen_params_t;
 
 typedef struct {
@@ -443,6 +455,7 @@ typedef struct {
     sd_hires_params_t hires;
     bool circular_x;
     bool circular_y;
+    sd_image_preprocess_params_t image_preprocess;
 } sd_vid_gen_params_t;
 
 typedef struct sd_ctx_t sd_ctx_t;
@@ -451,6 +464,11 @@ struct ggml_tensor;
 typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data);
 typedef void (*sd_progress_cb_t)(int step, int steps, float time, void* data);
 typedef void (*sd_preview_cb_t)(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data);
+
+typedef struct {
+    int sample_pass;
+    int total_steps;
+} sd_preview_info_t;
 typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data);
 
 SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
@@ -459,11 +477,17 @@ SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
 // negative interval previews only completed logical step -interval. Zero previews the final
 // completed step of the first sampling pass (base-resolution or high-noise).
 SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data);
+// Query from a preview callback. Passes are numbered from 1 since the last
+// sd_set_preview_callback call; total_steps is the actual count in that pass.
+// Both fields are zero before sampling starts.
+SD_API sd_preview_info_t sd_get_preview_info();
 SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data);
 SD_API int32_t sd_get_num_physical_cores();
 SD_API const char* sd_get_system_info();
 SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
 SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
+/** Returns whether the loaded model consumes ref_images as conditioning inputs. */
+SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx);
 
 // ControlNet hot-swap APIs are not safe to call while generation is in flight.
 SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path);
@@ -550,6 +574,8 @@ SD_API bool upscale(upscaler_ctx_t* upscaler_ctx,
                     int* num_images_out);
 
 SD_API int get_upscale_factor(upscaler_ctx_t* upscaler_ctx);
+// Reads model metadata only; returns 0 if the file is not a recognized RGB ESRGAN model.
+SD_API int get_upscaler_model_scale(const char* model_path);
 
 typedef struct adetailer_ctx_t adetailer_ctx_t;
 
@@ -589,7 +615,9 @@ SD_API bool convert_with_components(const char* model_path,
                                     enum sd_type_t output_type,
                                     const char* tensor_type_rules,
                                     bool convert_name,
-                                    int n_threads);
+                                    int n_threads,
+                                    const sd_lora_t* loras,
+                                    int lora_count);
 
 SD_API bool preprocess_canny(sd_image_t image,
                              float high_threshold,
diff --git a/scripts/format-code.ps1 b/scripts/format-code.ps1
index 195c5528d1..91c5f3b03a 100644
--- a/scripts/format-code.ps1
+++ b/scripts/format-code.ps1
@@ -8,6 +8,8 @@ $patterns = @(
     "src/core/*.cpp"
     "src/core/*.h"
     "src/core/*.hpp"
+    "src/core/probe/*.cpp"
+    "src/core/probe/*.h"
     "src/extensions/*.cpp"
     "src/extensions/*.h"
     "src/extensions/*.hpp"
diff --git a/scripts/format-code.sh b/scripts/format-code.sh
index f5e1d88751..568a0e54bb 100644
--- a/scripts/format-code.sh
+++ b/scripts/format-code.sh
@@ -8,6 +8,7 @@ cd "$repo_root"
 for f in src/*.cpp src/*.h src/*.hpp \
          src/conditioning/*.cpp src/conditioning/*.h src/conditioning/*.hpp \
          src/core/*.cpp src/core/*.h src/core/*.hpp \
+         src/core/probe/*.cpp src/core/probe/*.h \
          src/extensions/*.cpp src/extensions/*.h src/extensions/*.hpp \
          src/pipeline/*.cpp src/pipeline/*.h \
          src/runtime/*.cpp src/runtime/*.h src/runtime/*.hpp \
diff --git a/scripts/layer_split_ranges_test.cpp b/scripts/layer_split_ranges_test.cpp
new file mode 100644
index 0000000000..171ed8bec5
--- /dev/null
+++ b/scripts/layer_split_ranges_test.cpp
@@ -0,0 +1,86 @@
+// Standalone check of the layer-split range planner: g++ -std=c++17 -I src scripts/layer_split_ranges_test.cpp && ./a.out
+#include <cstdio>
+#include <cstdlib>
+
+#include "core/layer_split_ranges.h"
+
+static int failures = 0;
+#define CHECK(cond)                                                     \
+    do {                                                                \
+        if (!(cond)) {                                                  \
+            std::fprintf(stderr, "FAIL %s:%d %s\n", __FILE__, __LINE__, #cond); \
+            failures++;                                                 \
+        }                                                               \
+    } while (0)
+
+static bool contiguous(const sd::LayerSplitRangesResult& r) {
+    for (size_t i = 1; i < r.device_by_segment.size(); i++) {
+        if (r.device_by_segment[i] < r.device_by_segment[i - 1]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+int main() {
+    const int64_t MB = 1024ll * 1024;
+    // 54 equal segments, 10.98 GB total, two cards with 12.6 GB weight capacity each (16 GB card budget less headroom).
+    std::vector<int64_t> segs(54, (int64_t)(10.98 * 1024 * MB / 54));
+    std::vector<int64_t> caps = {12599 * MB, 12599 * MB};
+    auto fill                 = sd::plan_layer_split_ranges_fill_first(segs, caps);
+    CHECK(fill.ok);
+    CHECK(fill.bytes_by_device[1] == 0);  // the old plan: everything on card 0
+    auto prop = sd::plan_layer_split_ranges(segs, caps);
+    CHECK(prop.ok);
+    CHECK(!prop.used_fill_first);
+    CHECK(contiguous(prop));
+    CHECK(prop.device_by_segment.front() == 0 && prop.device_by_segment.back() == 1);
+    CHECK(std::llabs(prop.bytes_by_device[0] - prop.bytes_by_device[1]) <= segs[0]);
+
+    // Unequal capacities: shares follow the capacities.
+    caps = {4000 * MB, 12000 * MB};
+    prop = sd::plan_layer_split_ranges(segs, caps);
+    CHECK(prop.ok && contiguous(prop));
+    CHECK(prop.bytes_by_device[0] <= caps[0] && prop.bytes_by_device[1] <= caps[1]);
+    CHECK(prop.bytes_by_device[0] < prop.bytes_by_device[1]);
+
+    // Exactly full: proportional must still fit (falls back to fill-first if rounding misses).
+    std::vector<int64_t> four(4, 100);
+    caps = {200, 200};
+    prop = sd::plan_layer_split_ranges(four, caps);
+    CHECK(prop.ok && prop.bytes_by_device[0] == 200 && prop.bytes_by_device[1] == 200);
+
+    // Does not fit at all: failure names the last device.
+    caps = {150, 150};
+    prop = sd::plan_layer_split_ranges(four, caps);
+    CHECK(!prop.ok);
+    CHECK(prop.failed_device == 1);
+
+    // Uneven segment sizes where the proportional plan overflows: fill-first rescues it.
+    std::vector<int64_t> uneven = {10, 10, 10, 70};
+    caps                        = {30, 70};
+    prop                        = sd::plan_layer_split_ranges(uneven, caps);
+    CHECK(prop.ok);
+    CHECK(prop.bytes_by_device[0] == 30 && prop.bytes_by_device[1] == 70);
+
+    // Unknown capacity sentinel (no free-memory report): still splits evenly.
+    caps = {INT64_MAX / 4, INT64_MAX / 4};
+    prop = sd::plan_layer_split_ranges(segs, caps);
+    CHECK(prop.ok && prop.bytes_by_device[1] > 0);
+
+    // Zero-byte segments inherit the current device and never trigger a move.
+    std::vector<int64_t> zeros = {0, 50, 0, 50, 0};
+    caps                       = {60, 60};
+    prop                       = sd::plan_layer_split_ranges(zeros, caps);
+    CHECK(prop.ok && contiguous(prop) && prop.bytes_by_device[0] == 50 && prop.bytes_by_device[1] == 50);
+
+    // Single device: same as fill-first.
+    caps = {INT64_MAX / 4};
+    prop = sd::plan_layer_split_ranges(segs, caps);
+    CHECK(prop.ok && prop.bytes_by_device.size() == 1);
+
+    if (failures == 0) {
+        std::printf("layer_split_ranges_test: all checks passed\n");
+    }
+    return failures == 0 ? 0 : 1;
+}
diff --git a/scripts/unsloth/cpu_variants_gate.sh b/scripts/unsloth/cpu_variants_gate.sh
new file mode 100755
index 0000000000..e8cb8ca1bc
--- /dev/null
+++ b/scripts/unsloth/cpu_variants_gate.sh
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: MIT
+# Copyright 2026-present the Unsloth AI Inc. team.
+#
+#   pin   <bin_dir>: Linux: DT_RPATH=$ORIGIN (beats LD_LIBRARY_PATH, unlike RUNPATH) so the bundle's ggml wins.
+#   check <bundle.zip> <floor_variant>: floor variant present, no build-tree paths, sd-cli loads a CPU variant from the bundle.
+
+set -euo pipefail
+
+mode="${1:?usage: cpu_variants_gate.sh pin <bin_dir> | check <bundle.zip> <floor_variant>}"
+
+case "$mode" in
+  pin)
+    bin_dir="${2:?bin_dir}"
+    [ "$(uname -s)" = "Linux" ] || { echo "pin: nothing to do on $(uname -s)"; exit 0; }
+    command -v patchelf > /dev/null || { echo "ERROR: patchelf not installed (the workflow installs it)" >&2; exit 1; }
+    n=0
+    for f in "$bin_dir"/sd-cli "$bin_dir"/sd-server "$bin_dir"/libggml*.so* "$bin_dir"/libstable-diffusion*.so*; do
+      [ -f "$f" ] && [ ! -L "$f" ] || continue
+      patchelf --force-rpath --set-rpath '$ORIGIN' "$f"
+      n=$((n + 1))
+    done
+    echo "pin: set DT_RPATH=\$ORIGIN on $n files"
+    ;;
+  check)
+    zip="${2:?bundle.zip}"
+    floor="${3:?floor variant, e.g. x64 or armv8.0_1}"
+    work=""
+    empty=""
+    cleanup() {
+      if [ -n "$work" ]; then rm -rf -- "$work"; fi
+      if [ -n "$empty" ]; then rm -rf -- "$empty"; fi
+    }
+    trap cleanup EXIT
+    work="$(mktemp -d "${RUNNER_TEMP:-/tmp}/cpu-variants-XXXXXX")"
+    unzip -q "$zip" -d "$work"
+    dir="$(dirname "$(find "$work" -type f \( -name sd-cli -o -name sd-cli.exe \) | head -n1)")"
+    [ -n "$dir" ] && [ -d "$dir" ] || { echo "ERROR: no sd-cli in $zip" >&2; exit 1; }
+    case "$(uname -s)" in
+      Darwin) ext=so ;;  # ggml names its MODULE libraries .so on macOS too
+      MINGW*|MSYS*|CYGWIN*) ext=dll ;;
+      *) ext=so ;;
+    esac
+    prefix="libggml-cpu-"
+    [ "$ext" = dll ] && prefix="ggml-cpu-"
+    variants=""
+    for f in "$dir"/"$prefix"*."$ext"; do
+      [ -f "$f" ] || continue
+      v="${f##*/}"
+      v="${v#"$prefix"}"
+      variants="$variants${v%."$ext"} "
+    done
+    echo "variants: $variants"
+    case " $variants " in
+      *" $floor "*) ;;
+      *) echo "ERROR: floor variant '$floor' missing; hosts below the next level would have no CPU backend" >&2; exit 1 ;;
+    esac
+    fail=0
+    if [ "$(uname -s)" = "Linux" ]; then
+      for f in "$dir"/sd-cli "$dir"/sd-server "$dir"/libggml*.so* "$dir"/libstable-diffusion*.so*; do
+        [ -f "$f" ] || continue
+        rp="$(readelf -d "$f" | sed -n 's/.*(\(RPATH\|RUNPATH\)).*\[\(.*\)\]/\1=\2/p')"
+        if [ "$rp" != 'RPATH=$ORIGIN' ]; then echo "ERROR: $(basename "$f") has '${rp:-no rpath}', want RPATH=\$ORIGIN" >&2; fail=1; fi
+      done
+    elif [ "$(uname -s)" = "Darwin" ]; then
+      for f in "$dir"/sd-cli "$dir"/sd-server "$dir"/*.dylib "$dir"/*.so; do
+        [ -f "$f" ] || continue
+        bad="$(otool -l "$f" | awk '/LC_RPATH/{getline; getline; print $2}' | grep -v '^@' || true)"
+        if [ -n "$bad" ]; then echo "ERROR: $(basename "$f") keeps a build-tree rpath: $bad" >&2; fail=1; fi
+        deps="$(otool -L "$f" | tail -n +2 | awk '{print $1}' | grep -E 'ggml|stable-diffusion' | grep -v '^@' || true)"
+        if [ -n "$deps" ]; then echo "ERROR: $(basename "$f") links ggml by absolute path: $deps" >&2; fail=1; fi
+      done
+    fi
+    cli="$dir/sd-cli"
+    [ -f "$cli.exe" ] && cli="$cli.exe"
+    # Nonexistent model: sd-cli loads backends then stops. Empty cwd defeats ggml's cwd search.
+    empty="$(mktemp -d "${RUNNER_TEMP:-/tmp}/cpu-variants-cwd-XXXXXX")"
+    out="$(cd "$empty" && env -u LD_LIBRARY_PATH -u DYLD_LIBRARY_PATH "$cli" -m "$empty/none.gguf" -p x -v 2>&1 || true)"
+    loaded="$(printf '%s\n' "$out" | sed -n 's/.*loaded CPU backend from \(.*\)$/\1/p' | head -n1 | tr -d '\r')"
+    printf '%s\n' "$out" | grep -A1 "System Info" | head -n 3 || true
+    if [ -z "$loaded" ]; then
+      echo "ERROR: sd-cli loaded no CPU backend variant" >&2
+      printf '%s\n' "$out" | tail -n 30 >&2
+      fail=1
+    else
+      echo "loaded: $loaded"
+      loaded_dir="$(cd "$(dirname "$loaded")" && pwd -P)"
+      want_dir="$(cd "$dir" && pwd -P)"
+      if [ "$loaded_dir" != "$want_dir" ]; then echo "ERROR: variant came from $loaded_dir, not the bundle" >&2; fail=1; fi
+    fi
+    exit "$fail"
+    ;;
+  *)
+    echo "unknown mode $mode" >&2
+    exit 2
+    ;;
+esac
diff --git a/scripts/unsloth/ggml-patches/0001-ggml-cuda-h3-speed.patch b/scripts/unsloth/ggml-patches/0001-ggml-cuda-h3-speed.patch
new file mode 100644
index 0000000000..bac37e747f
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0001-ggml-cuda-h3-speed.patch
@@ -0,0 +1,233 @@
+From fa18cddc3f2a87bec12af3a00d4be0e28f9d514b Mon Sep 17 00:00:00 2001
+From: Daniel Han <danielhanchen@gmail.com>
+Date: Wed, 30 Sep 2026 22:28:21 -0700
+Subject: [PATCH 1/2] ggml-cuda: BF16 cuBLAS for large-batch quantized matmul,
+ cache device integrated flag
+
+GGML_CUDA_QUANT_CUBLAS_MIN_BATCH=<rows> dequantizes a quantized weight to BF16
+and runs a cuBLAS GEMM (FP32 accumulate, FP32 output) instead of MMQ when src1
+has at least that many rows; 0 or unset keeps MMQ everywhere.
+GGML_CUDA_QUANT_CUBLAS_TYPE=f32 dequantizes to FP32 instead (accuracy oracle).
+At diffusion-transformer sizes (MiniMax-H3 960x544x124: 19108 tokens per call)
+MMQ reaches 130-240 TFLOP/s on a B200 while the BF16 GEMM reaches ~1300, and
+the result is closer to an FP32 reference because activations stay BF16
+instead of being quantized to q8_1.
+
+ggml_backend_cuda_device_get_memory / get_type called cudaGetDeviceProperties
+(~1-2 ms) on every call; stable-diffusion.cpp polls them per graph, 2798 times
+in one tiled H3 VAE decode. The flag is now read once at registration.
+---
+ src/ggml-cuda/ggml-cuda.cu | 55 ++++++++++++++++++++++++++++++++------
+ 1 file changed, 47 insertions(+), 8 deletions(-)
+
+[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is
+ the whole patch as one diff, regenerated from the port.]
+---
+diff --git a/src/ggml-cuda/cpy.cu b/src/ggml-cuda/cpy.cu
+index 5689f21b..a85d84bc 100644
+--- a/src/ggml-cuda/cpy.cu
++++ b/src/ggml-cuda/cpy.cu
+@@ -1,4 +1,5 @@
+ #include "cpy.cuh"
++#include <type_traits>
+ #include "dequantize.cuh"
+ #include "cpy-utils.cuh"
+ #include "fp8.cuh"
+@@ -203,6 +204,84 @@ cudaStream_t stream) {
+     ggml_cuda_kernel_launch(cpy_scalar_contiguous<src_t, dst_t>, launch_params, cx, cdst, ne);
+ }
+ 
++// Strided copy whose rows (dim 0) are contiguous on both sides and of equal length: one float4 per thread, the
++// 4-D index math done once per row instead of per element. cpy_scalar spends most of its time in the 64-bit
++// div/mod of that per-element math; the permuted ggml_cont copies of a DiT's Q/K/V (rows of head_dim floats) run
++// at a few percent of memory bandwidth through it. A pure copy (or the same per-element conversion), so the
++// result is bit-identical to cpy_scalar. GGML_CUDA_CPY_ROWS=0 disables it.
++template <typename dst_t>
++static __global__ void cpy_rows_vec4(const char * cx, char * cdst, const int64_t nrows, const int64_t ne00_4,
++                                     const int64_t ne01, const int64_t ne02, const int64_t nb01, const int64_t nb02, const int64_t nb03,
++                                     const int64_t ne11, const int64_t ne12, const int64_t nb11, const int64_t nb12, const int64_t nb13) {
++    ggml_cuda_pdl_lc();
++    const int64_t row = (int64_t) blockIdx.x*blockDim.y + threadIdx.y;
++    if (row >= nrows) {
++        return;
++    }
++    const int64_t i01 = row % ne01;
++    const int64_t i02 = (row / ne01) % ne02;
++    const int64_t i03 = row / (ne01*ne02);
++    const int64_t i11 = row % ne11;
++    const int64_t i12 = (row / ne11) % ne12;
++    const int64_t i13 = row / (ne11*ne12);
++    const float4 * src = (const float4 *) (cx + i01*nb01 + i02*nb02 + i03*nb03);
++    char * dst_row = cdst + i11*nb11 + i12*nb12 + i13*nb13;
++    ggml_cuda_pdl_sync();
++    for (int64_t c = threadIdx.x; c < ne00_4; c += blockDim.x) {
++        const float4 v = src[c];
++        if constexpr (std::is_same_v<dst_t, float>) {
++            ((float4 *) dst_row)[c] = v;
++        } else {
++            dst_t * d = (dst_t *) dst_row + 4*c;
++            cpy_1_scalar<float, dst_t>((const char *) &v.x, (char *) (d + 0));
++            cpy_1_scalar<float, dst_t>((const char *) &v.y, (char *) (d + 1));
++            cpy_1_scalar<float, dst_t>((const char *) &v.z, (char *) (d + 2));
++            cpy_1_scalar<float, dst_t>((const char *) &v.w, (char *) (d + 3));
++        }
++    }
++}
++
++static bool ggml_cuda_cpy_rows_enabled() {
++    static const bool enabled = [] {
++        const char * e = getenv("GGML_CUDA_CPY_ROWS");
++        return e == nullptr || atoi(e) != 0;
++    }();
++    return enabled;
++}
++
++// True (and launched) when src0 -> src1 is a row copy cpy_rows_vec4 can do; false leaves the copy to the caller.
++template <typename dst_t>
++static bool ggml_cpy_rows_vec4_cuda(const ggml_tensor * src0, ggml_tensor * src1, cudaStream_t stream) {
++    if (!ggml_cuda_cpy_rows_enabled() || src0->type != GGML_TYPE_F32) {
++        return false;
++    }
++    const int64_t ne00 = src0->ne[0];
++    if (src0->nb[0] != (int64_t) sizeof(float) || src1->nb[0] != (int64_t) sizeof(dst_t) || src1->ne[0] != ne00 || ne00 % 4 != 0) {
++        return false;
++    }
++    const size_t dst_align = std::is_same_v<dst_t, float> ? 16 : 2*sizeof(dst_t);
++    if (((uintptr_t) src0->data | src0->nb[1] | src0->nb[2] | src0->nb[3]) % 16 != 0 ||
++        ((uintptr_t) src1->data | src1->nb[1] | src1->nb[2] | src1->nb[3]) % dst_align != 0) {
++        return false;
++    }
++    const int64_t nrows = ggml_nrows(src0);
++    if (nrows != ggml_nrows(src1) || src0->ne[1] == 0 || src0->ne[2] == 0 || src1->ne[1] == 0 || src1->ne[2] == 0) {
++        return false;
++    }
++    const int64_t ne00_4 = ne00 / 4;
++    const int threads_x = ne00_4 >= 32 ? 32 : (int) ne00_4;
++    const int threads_y = 256 / threads_x;
++    const int64_t blocks = (nrows + threads_y - 1) / threads_y;
++    if (blocks > INT_MAX) {
++        return false;
++    }
++    const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(dim3((unsigned) blocks), dim3(threads_x, threads_y), 0, stream);
++    ggml_cuda_kernel_launch(cpy_rows_vec4<dst_t>, launch_params, (const char *) src0->data, (char *) src1->data, nrows, ne00_4,
++        src0->ne[1], src0->ne[2], (int64_t) src0->nb[1], (int64_t) src0->nb[2], (int64_t) src0->nb[3],
++        src1->ne[1], src1->ne[2], (int64_t) src1->nb[1], (int64_t) src1->nb[2], (int64_t) src1->nb[3]);
++    return true;
++}
++
+ template<typename src_t, typename dst_t, bool transposed = false>
+ static void ggml_cpy_scalar_cuda(
+     const char * cx, char * cdst, const int64_t ne,
+@@ -510,6 +589,10 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
+             ggml_cpy_scalar_cuda<ggml_fp8_e5m2_cuda, nv_bfloat16>
+                 (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+         }
++    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && ggml_cpy_rows_vec4_cuda<float>(src0, src1, main_stream)) {
++        // row copy done
++    } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16 && !contiguous_srcs && ggml_cpy_rows_vec4_cuda<half>(src0, src1, main_stream)) {
++        // row copy with conversion done
+     } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {
+         if (can_be_transposed) {
+             ggml_cpy_scalar_cuda<float, float, true>
+diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu
+index c28589ae..8a295c19 100644
+--- a/src/ggml-cuda/ggml-cuda.cu
++++ b/src/ggml-cuda/ggml-cuda.cu
+@@ -2561,6 +2561,25 @@ static bool ggml_cuda_match_shared_expert(const ggml_cgraph * graph, int routed_
+     return true;
+ }
+ 
++// Large-batch matmul with a quantized weight: dequantize the weight to BF16 once and run a cuBLAS
++// GEMM (FP32 accumulate, FP32 output) instead of MMQ. MMQ quantizes the activations to q8_1 and runs
++// int8 MMA, which is the right call for LLM prompt sizes, but at diffusion-transformer sizes (10^4
++// tokens per call) a tensor-core BF16 GEMM is faster on GPUs whose BF16 rate is not far below their
++// int8 rate, and it is more accurate (activations stay BF16 instead of 8-bit).
++// GGML_CUDA_QUANT_CUBLAS_MIN_BATCH=<rows>: use the BF16 path when src1 has at least this many rows;
++// 0 disables it (stock MMQ everywhere). Unset = the per-architecture default below.
++static int64_t ggml_cuda_quant_cublas_min_batch(int cc) {
++    static const int64_t env = [] {
++        const char * e = getenv("GGML_CUDA_QUANT_CUBLAS_MIN_BATCH");
++        return e != nullptr ? (int64_t) atoll(e) : (int64_t) -1;
++    }();
++    if (env >= 0) {
++        return env;
++    }
++    GGML_UNUSED(cc);
++    return 0;
++}
++
+ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
+     GGML_TENSOR_BINARY_OP_LOCALS
+ 
+@@ -2625,6 +2644,23 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
+         ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst);
+         return;
+     }
++    if (ggml_is_quantized(src0->type) && ne11 > MMVQ_MAX_BATCH_SIZE && GGML_CUDA_CC_IS_NVIDIA(cc) &&
++            bf16_mma_hardware_available(cc) && ggml_get_to_bf16_cuda(src0->type) != nullptr) {
++        const int64_t min_batch = ggml_cuda_quant_cublas_min_batch(cc);
++        if (min_batch > 0 && ne11*ne12*ne13 >= min_batch) {
++            // GGML_CUDA_QUANT_CUBLAS_TYPE=f32 dequantizes to FP32 instead: slow, for accuracy oracles only.
++            static const bool f32_oracle = [] {
++                const char * e = getenv("GGML_CUDA_QUANT_CUBLAS_TYPE");
++                return e != nullptr && (strcmp(e, "f32") == 0 || strcmp(e, "fp32") == 0);
++            }();
++            if (f32_oracle) {
++                ggml_cuda_mul_mat_cublas_impl<GGML_TYPE_F32>(ctx, src0, src1, dst);
++            } else {
++                ggml_cuda_mul_mat_cublas_impl<GGML_TYPE_BF16>(ctx, src0, src1, dst);
++            }
++            return;
++        }
++    }
+     if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) {
+         ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst);
+         return;
+@@ -5881,6 +5917,10 @@ struct ggml_backend_cuda_device_context {
+     std::string description;
+     std::string pci_bus_id;
+     int op_offload_min_batch_size;
++    // cudaDeviceProp::integrated, read once at registration. cudaGetDeviceProperties costs ~1-2 ms per call, and
++    // get_memory / get_type are polled per graph by callers that budget VRAM (stable-diffusion.cpp asks thousands
++    // of times per tiled VAE decode, ~5 s of host time per H3 video on a B200 host).
++    bool integrated = false;
+ };
+ 
+ static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) {
+@@ -5984,11 +6024,8 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
+ // ref: https://github.com/ggml-org/llama.cpp/pull/17368
+ #if defined(__linux__) && !defined(GGML_USE_HIP)
+     // Check if this is a UMA (Unified Memory Architecture) system
+-    cudaDeviceProp prop;
+-    CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device)));
+-
+     // Check if UMA is explicitly enabled via environment variable
+-    bool is_uma = prop.integrated > 0 || ggml_cuda_unified_memory_enabled();
++    bool is_uma = ctx->integrated || ggml_cuda_unified_memory_enabled();
+ 
+     if (is_uma) {
+         // For UMA systems (like DGX Spark), use system memory info
+@@ -6012,10 +6049,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t *
+ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend_dev_t dev) {
+     ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *) dev->context;
+ 
+-    cudaDeviceProp prop;
+-    CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device)));
+-
+-    return prop.integrated
++    return ctx->integrated
+         ? GGML_BACKEND_DEVICE_TYPE_IGPU
+         : GGML_BACKEND_DEVICE_TYPE_GPU;
+ }
+@@ -6795,6 +6829,11 @@ ggml_backend_reg_t ggml_backend_cuda_reg() {
+                 dev_ctx->device = i;
+                 dev_ctx->name = GGML_CUDA_NAME + std::to_string(i);
+                 dev_ctx->description = ggml_cuda_device_description(i);
++                {
++                    cudaDeviceProp prop;
++                    CUDA_CHECK(cudaGetDeviceProperties(&prop, physical_id));
++                    dev_ctx->integrated = prop.integrated > 0;
++                }
+ 
+                 char pci_bus_id[32] = {};
+                 CUDA_CHECK(cudaDeviceGetPCIBusId(pci_bus_id, sizeof(pci_bus_id), physical_id));
diff --git a/scripts/unsloth/ggml-patches/0002-ggml-cuda-fa-longseq.patch b/scripts/unsloth/ggml-patches/0002-ggml-cuda-fa-longseq.patch
new file mode 100644
index 0000000000..190aac4918
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0002-ggml-cuda-fa-longseq.patch
@@ -0,0 +1,433 @@
+From dc60887c4e3eec372301d069f4dd90874745dc91 Mon Sep 17 00:00:00 2001
+From: Daniel Han <danielhanchen@gmail.com>
+Date: Thu, 1 Oct 2026 05:41:03 -0700
+Subject: [PATCH 1/3] ggml-cuda: pipelined long-sequence FlashAttention for
+ unmasked attention without GQA packing
+
+The mma FlashAttention kernels without GQA packing (ncols2 == 1, used whenever
+there is no mask) load every K/V tile synchronously with an out-of-bounds check,
+because the KV length need not be a multiple of the KV tile. For diffusion
+transformers (no mask, 10^4 tokens) the kernel then spends most of its time
+waiting on global memory.
+
+The long-sequence variant pipelines all full KV tiles with cp.async (the
+existing two-stage path) and loads only the final, partial tile synchronously
+with the bounds check. Per output element the arithmetic and its order are
+unchanged, so results are bit-identical to the stock kernel.
+
+It is used for head sizes 64 and 128 with K == V head size, no mask, no sinks,
+no logit softcap and KV length >= 256, on NVIDIA GPUs with cp.async (Ampere and
+newer). GGML_CUDA_FA_LONGSEQ=0 restores the stock kernels.
+
+GGML_CUDA_FA_LONGSEQ_NCOLS=128 additionally selects a 128-column tile with 8
+warps for Q length >= 1024 (stream-k splits then differ, so results are not
+bit-identical); off by default.
+
+test-backend-ops gains unmasked FLASH_ATTN_EXT cases with partial last KV
+tiles, several Q tiles and GQA broadcast.
+---
+ src/ggml-cuda/fattn-mma-f16.cuh               | 167 ++++++++++++++++--
+ src/ggml-cuda/fattn.cu                        |   9 +
+ ...tn-mma-f16-instance-ncols1_128-ncols2_1.cu |   8 +
+ tests/test-backend-ops.cpp                    |  15 ++
+ 4 files changed, 184 insertions(+), 15 deletions(-)
+ create mode 100644 src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_128-ncols2_1.cu
+
+[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is
+ the whole patch as one diff, regenerated from the port.]
+---
+diff --git a/src/ggml-cuda/fattn-common.cuh b/src/ggml-cuda/fattn-common.cuh
+index 41d94833..82f43137 100644
+--- a/src/ggml-cuda/fattn-common.cuh
++++ b/src/ggml-cuda/fattn-common.cuh
+@@ -981,7 +981,10 @@ template <int DV, int ncols1, int ncols2>
+ void launch_fattn(
+     ggml_backend_cuda_context & ctx, ggml_tensor * dst, fattn_kernel_t fattn_kernel, const int nwarps, const size_t nbytes_shared,
+     const int nbatch_fa, const bool need_f16_K, const bool need_f16_V, const bool stream_k, const bool use_sparse,
+-    const int warp_size = WARP_SIZE, const bool async_kv_preload = false
++    const int warp_size = WARP_SIZE, const bool async_kv_preload = false,
++    // Optional: size the grid from the occupancy of this kernel instead of fattn_kernel. A drop-in kernel variant launched
++    // with the grid of the kernel it replaces splits the work (stream-k seams) identically and so gives identical results.
++    fattn_kernel_t occupancy_kernel = nullptr, const size_t occupancy_nbytes_shared = 0
+ ) {
+     constexpr int ncols = ncols1 * ncols2;
+ 
+@@ -1134,7 +1137,8 @@ void launch_fattn(
+ 
+     const dim3 block_dim(warp_size, nwarps, 1);
+     int max_blocks_per_sm = 1; // Max. number of active blocks limited by occupancy.
+-    CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, fattn_kernel, block_dim.x * block_dim.y * block_dim.z, nbytes_shared));
++    CUDA_CHECK(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_sm, occupancy_kernel ? occupancy_kernel : fattn_kernel,
++        block_dim.x * block_dim.y * block_dim.z, occupancy_kernel ? occupancy_nbytes_shared : nbytes_shared));
+     GGML_ASSERT(max_blocks_per_sm > 0);
+     int parallel_blocks = max_blocks_per_sm;
+ 
+diff --git a/src/ggml-cuda/fattn-mma-f16.cuh b/src/ggml-cuda/fattn-mma-f16.cuh
+index 296aa2b3..e011c8d5 100644
+--- a/src/ggml-cuda/fattn-mma-f16.cuh
++++ b/src/ggml-cuda/fattn-mma-f16.cuh
+@@ -61,6 +61,10 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
+     GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 32, 128, 2,  64,  64,  64,  64, 2, true);
+     GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 64, 128, 2,  64,  64,  64,  64, 2, true);
+ 
++    // ncols == 128 is only used by the long-sequence variant (ncols2 == 1, no mask), see ggml_cuda_fattn_mma_use_longseq:
++    GGML_CUDA_FATTN_MMA_CONFIG_CASE( 64,  64, 128, 256, 1,  64,  32,  32,  32, 2, true);
++    GGML_CUDA_FATTN_MMA_CONFIG_CASE(128, 128, 128, 256, 1,  64,  64,  64,  64, 2, true);
++
+     GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128,  8,  64, 4,  64,  96,  64,  64, 2, true);
+     GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 16,  64, 4,  32,  96,  64,  64, 2, true);
+     GGML_CUDA_FATTN_MMA_CONFIG_CASE(192, 128, 32, 128, 2,  32,  96,  64,  64, 2, true);
+@@ -378,6 +382,24 @@ static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages(
+ #endif // CP_ASYNC_AVAILABLE
+ }
+ 
++// Long-sequence variant of the kernels without GQA packing (ncols2 == 1) and without a mask, used for diffusion models.
++// The stock ncols2 == 1 kernels load K/V synchronously with an out-of-bounds check on every KV tile because the
++// KV length need not be a multiple of nbatch_fa. The long-sequence variant pipelines all full KV tiles with cp.async
++// and only loads the final, partial tile synchronously with the OOB check. The arithmetic per output element is the
++// same as in the stock kernel, so the results are bit-identical.
++static __host__ int ggml_cuda_fattn_mma_get_nstages_longseq(const int DKQ, const int DV, const int ncols, const int cc) {
++    return cp_async_available(cc) ? ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols, cc) : 0;
++}
++
++static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages_longseq(const int DKQ, const int DV, const int ncols) {
++#ifdef CP_ASYNC_AVAILABLE
++    return ggml_cuda_fattn_mma_get_nstages_target(DKQ, DV, ncols);
++#else
++    GGML_UNUSED_VARS(DKQ, DV, ncols);
++    return 0;
++#endif // CP_ASYNC_AVAILABLE
++}
++
+ // ------------------------------------------------------------------------------------------------------------------
+ 
+ template<int stride_tile, int nwarps, int nbatch_fa, bool use_cp_async, bool oob_check, bool use_sparse>
+@@ -573,7 +595,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_load_mask(
+ 
+ template<int DKQ, int DV, int ncols1, int ncols2, int nwarps,
+     bool use_logit_softcap, bool V_is_K_view, bool use_sparse, bool needs_fixup, bool is_fixup, bool last_iter, bool oob_check,
+-    typename T_A_KQ, typename T_B_KQ, typename T_C_KQ, typename T_A_VKQ, typename T_B_VKQ, typename T_C_VKQ>
++    typename T_A_KQ, typename T_B_KQ, typename T_C_KQ, typename T_A_VKQ, typename T_B_VKQ, typename T_C_VKQ, int nstages_override = -1>
+ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
+         const float2 * const __restrict__ Q_f2,
+         const half2  * const __restrict__ K_h2,
+@@ -611,7 +633,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
+     constexpr int  nbatch_K2       = ggml_cuda_fattn_mma_get_nbatch_K2(DKQ, DV, ncols);
+     constexpr int  nbatch_V2       = ggml_cuda_fattn_mma_get_nbatch_V2(DKQ, DV, ncols);
+     constexpr bool Q_in_reg        = ggml_cuda_fattn_mma_get_Q_in_reg (DKQ, DV, ncols);
+-    constexpr int  nstages         = ggml_cuda_fattn_mma_get_nstages  (DKQ, DV, ncols1, ncols2, use_sparse);
++    constexpr int  nstages         = nstages_override >= 0 ? nstages_override : ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, use_sparse);
+ 
+     constexpr bool swz = ggml_cuda_fattn_mma_get_swizzled(DKQ, DV, ncols);
+     constexpr int stride_tile_K = swz ? nbatch_K2 : nbatch_K2 + 4;
+@@ -1186,7 +1208,8 @@ template<int DKQ, int ncols> struct mma_tile_sizes {
+ };
+ #endif // defined(TURING_MMA_AVAILABLE)
+ 
+-template<int DKQ, int DV, int ncols1, int ncols2, int nwarps, bool use_logit_softcap, bool V_is_K_view, bool use_sparse, bool needs_fixup, bool is_fixup>
++template<int DKQ, int DV, int ncols1, int ncols2, int nwarps, bool use_logit_softcap, bool V_is_K_view, bool use_sparse, bool needs_fixup, bool is_fixup,
++    bool longseq = false>
+ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
+         const float2 * const __restrict__ Q_f2,
+         const half2  * const __restrict__ K_h2,
+@@ -1232,7 +1255,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
+     constexpr int  nbatch_V2       = ggml_cuda_fattn_mma_get_nbatch_V2     (DKQ, DV, ncols);
+     constexpr int  nbatch_combine  = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols);
+     constexpr bool Q_in_reg        = ggml_cuda_fattn_mma_get_Q_in_reg      (DKQ, DV, ncols);
+-    constexpr int  nstages         = ggml_cuda_fattn_mma_get_nstages       (DKQ, DV, ncols1, ncols2, use_sparse);
++    constexpr int  nstages         = longseq ? ggml_cuda_fattn_mma_get_nstages_longseq(DKQ, DV, ncols) :
++                                               ggml_cuda_fattn_mma_get_nstages        (DKQ, DV, ncols1, ncols2, use_sparse);
++    static_assert(!longseq || (ncols2 == 1 && !V_is_K_view && !use_sparse), "bad longseq variant");
+ 
+     if (cols_per_warp > ncols) {
+         NO_DEVICE_CODE;
+@@ -1331,6 +1356,9 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
+ 
+     int kb0 = kb0_start;
+ 
++    // Long-sequence variant: only the last KV tile can extend past ne11, all tiles before it are pipelined.
++    const int kb0_stop_pipe = longseq && kb0_stop*nbatch_fa > ne11 ? kb0_stop - 1 : kb0_stop;
++
+     // Preload mask and K data for first iteration when using cp_async with multiple stages:
+     if constexpr (nstages > 1) {
+         static_assert(!use_sparse, "sparse gather not implemented for multi-stage loading");
+@@ -1338,16 +1366,54 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
+         constexpr bool use_cp_async = true;
+         constexpr bool oob_check    = false;
+         constexpr int  k_VKQ_sup    = nbatch_fa;
+-        if (ncols2 > 1 || mask_h) {
+-            flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
+-                (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
++        if (!longseq || kb0 < kb0_stop_pipe) {
++            if (ncols2 > 1 || mask_h) {
++                flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
++                    (mask_h, tile_mask, stride_mask, kb0*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
++            }
++            flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
++                (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr);
+         }
+-        flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
+-            (K_h2, tile_K, nbatch_K2, stride_K, kb0*nbatch_fa, k_VKQ_sup, nullptr);
+     }
+ 
+     // kb0_start is always < kb0_stop so the last iter can be executed unconditionally.
+-    if constexpr (ncols2 == 1 || use_sparse) {
++    if constexpr (longseq && nstages > 1) {
++        for (; kb0 < kb0_stop_pipe-1; ++kb0) {
++            constexpr bool last_iter = false;
++            constexpr bool oob_check = false;
++            constexpr int  k_VKQ_sup = nbatch_fa;
++            flash_attn_ext_f16_iter
++                <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
++                 T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, nstages>
++                (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
++                 ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
++                 KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
++        }
++        if (kb0 < kb0_stop_pipe) {
++            constexpr bool last_iter = true; // Do not preload past the pipelined tiles.
++            constexpr bool oob_check = false;
++            constexpr int  k_VKQ_sup = nbatch_fa;
++            flash_attn_ext_f16_iter
++                <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
++                 T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, nstages>
++                (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
++                 ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
++                 KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
++            ++kb0;
++        }
++        if (kb0 < kb0_stop) {
++            // Partial last tile: synchronous loads with an out-of-bounds check, as in the stock ncols2 == 1 kernel.
++            constexpr bool last_iter = true;
++            constexpr bool oob_check = true;
++            const     int  k_VKQ_sup = ne11 - kb0*nbatch_fa;
++            flash_attn_ext_f16_iter
++                <DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
++                 T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ, 0>
++                (Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
++                 ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
++                 KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
++        }
++    } else if constexpr (ncols2 == 1 || use_sparse) {
+         constexpr bool oob_check = true;
+         for (; kb0 < kb0_stop-1; ++kb0) {
+             constexpr bool last_iter = false;
+@@ -1793,7 +1859,7 @@ static constexpr __host__ __device__ bool ggml_cuda_flash_attn_ext_mma_f16_may_u
+            (DKQ == 256 && DV == 256 && ncols1 == 8 && ncols2 == 8);
+ }
+ 
+-template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view, bool use_sparse>
++template<int DKQ, int DV, int ncols1, int ncols2, bool use_logit_softcap, bool V_is_K_view, bool use_sparse, bool longseq = false>
+ __launch_bounds__(ggml_cuda_fattn_mma_get_nthreads(DKQ, DV, ncols1*ncols2), ggml_cuda_fattn_mma_get_occupancy(DKQ, DV, ncols1*ncols2))
+ static __global__ void flash_attn_ext_f16(
+         const char * Q_ptr,
+@@ -1936,12 +2002,12 @@ static __global__ void flash_attn_ext_f16(
+         constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
+         if (kb0_start == 0) {
+             constexpr bool needs_fixup = false; // CUDA block is working on an entire tile.
+-            flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
++            flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, longseq>
+                 (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
+                  ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
+         } else {
+             constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile.
+-            flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
++            flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, longseq>
+                 (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
+                  ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
+         }
+@@ -1985,7 +2051,7 @@ static __global__ void flash_attn_ext_f16(
+ 
+     constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
+     constexpr bool needs_fixup = false;
+-    flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
++    flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, longseq>
+         (Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
+          ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
+ #else
+@@ -2004,6 +2070,47 @@ static __global__ void flash_attn_ext_f16(
+ 
+ bool ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(const int cc, const ggml_tensor * dst, const int ncols1, const int ncols2);
+ 
++// Whether the long-sequence variant (see ggml_cuda_fattn_mma_get_nstages_longseq) can and should be used for dst.
++// GGML_CUDA_FA_LONGSEQ=0 disables it, =1 enables it on every NVIDIA GPU with cp.async, unset = per-architecture default.
++static bool ggml_cuda_fattn_mma_longseq_arch_default(const int cc) {
++    GGML_UNUSED(cc);
++    return true;
++}
++
++// Q columns per CUDA block of the long-sequence variant for long Q: 64 (the stock tile, default) or 128 (8 warps sharing
++// each K/V tile, half the K/V traffic per FLOP). 128 is opt-in via GGML_CUDA_FA_LONGSEQ_NCOLS=128: it was measured faster at
++// MiniMax-H3 DiT shapes on A100 (sm80, 75 -> 70 ms) and L4 (sm89, 460 -> 365 ms) but splits the work differently from the
++// stock kernel, so its results are not bit-identical.
++static int ggml_cuda_fattn_mma_longseq_ncols(const int cc, const int DKQ) {
++    static const int env = [] {
++        const char * e = getenv("GGML_CUDA_FA_LONGSEQ_NCOLS");
++        return e != nullptr ? atoi(e) : -1;
++    }();
++    GGML_UNUSED(cc);
++    GGML_UNUSED(DKQ);
++    return env == 128 ? 128 : 64;
++}
++
++static bool ggml_cuda_fattn_mma_use_longseq(const int cc, const ggml_tensor * dst) {
++    static const int env = [] {
++        const char * e = getenv("GGML_CUDA_FA_LONGSEQ");
++        return e != nullptr ? atoi(e) : -1;
++    }();
++    if (env == 0 || !GGML_CUDA_CC_IS_NVIDIA(cc) || !cp_async_available(cc)) {
++        return false;
++    }
++    const ggml_tensor * K     = dst->src[1];
++    const ggml_tensor * V     = dst->src[2];
++    const ggml_tensor * mask  = dst->src[3];
++    const ggml_tensor * sinks = dst->src[4];
++    float logit_softcap;
++    memcpy(&logit_softcap, (const float *) dst->op_params + 2, sizeof(float));
++    if (mask != nullptr || sinks != nullptr || logit_softcap != 0.0f || K->ne[0] != V->ne[0] || K->ne[1] < 256) {
++        return false;
++    }
++    return env == 1 || ggml_cuda_fattn_mma_longseq_arch_default(cc);
++}
++
+ template <int DKQ, int DV, int ncols1, int ncols2>
+ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+     const ggml_tensor * KQV = dst;
+@@ -2012,13 +2119,22 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
+ 
+     constexpr int ncols = ncols1 * ncols2;
+ 
++#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
++    constexpr bool longseq_instantiated = ncols2 == 1 && DKQ == DV && (DKQ == 64 || DKQ == 128);
++    const bool longseq = longseq_instantiated && ggml_cuda_fattn_mma_use_longseq(cc, dst);
++#else
++    constexpr bool longseq_instantiated = false;
++    const bool longseq = false;
++#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
++
+     const int  nthreads       = ggml_cuda_fattn_mma_get_nthreads      (DKQ, DV, ncols, cc);
+     const int  nbatch_fa      = ggml_cuda_fattn_mma_get_nbatch_fa     (DKQ, DV, ncols, cc);
+     const int  nbatch_K2      = ggml_cuda_fattn_mma_get_nbatch_K2     (DKQ, DV, ncols, cc);
+     const int  nbatch_V2      = ggml_cuda_fattn_mma_get_nbatch_V2     (DKQ, DV, ncols, cc);
+     const int  nbatch_combine = ggml_cuda_fattn_mma_get_nbatch_combine(DKQ, DV, ncols, cc);
+     const bool Q_in_reg       = ggml_cuda_fattn_mma_get_Q_in_reg      (DKQ, DV, ncols, cc);
+-    const int  nstages        = ggml_cuda_fattn_mma_get_nstages       (DKQ, DV, ncols1, ncols2, cc);
++    const int  nstages        = longseq ? ggml_cuda_fattn_mma_get_nstages_longseq(DKQ, DV, ncols, cc) :
++                                          ggml_cuda_fattn_mma_get_nstages        (DKQ, DV, ncols1, ncols2, cc);
+ 
+     const int cols_per_warp = std::min(ncols, get_cols_per_warp(cc));
+     const int warp_size_host = ggml_cuda_info().devices[ctx.device].warp_size;
+@@ -2052,6 +2168,40 @@ void ggml_cuda_flash_attn_ext_mma_f16_case(ggml_backend_cuda_context & ctx, ggml
+ #endif // defined(GGML_USE_HIP)
+     fattn_kernel_t fattn_kernel;
+     bool use_sparse = false;
++    if constexpr (longseq_instantiated) {
++        if (longseq) {
++            GGML_ASSERT(logit_softcap == 0.0f);
++            fattn_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, false, false, false, true>;
++
++            static bool shared_memory_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};
++            if (!shared_memory_limit_raised[id]) {
++                CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(fattn_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_shared_total));
++                shared_memory_limit_raised[id] = true;
++            }
++            if (ncols1 <= 64) {
++                // Same grid as the stock kernel of this tile size: identical stream-k seams, so results are bit-identical.
++                const int    nstages_stock      = ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, cc);
++                const size_t nbytes_KV_stock    = nstages_stock <= 1 ? nbytes_shared_KV_1stage : nbytes_shared_KV_2stage;
++                const size_t nbytes_total_stock = std::max(nbytes_shared_combine, Q_in_reg ?
++                    std::max(nbytes_shared_Q,  nbytes_KV_stock + nbytes_shared_mask) :
++                             nbytes_shared_Q + nbytes_KV_stock + nbytes_shared_mask);
++                fattn_kernel_t stock_kernel = flash_attn_ext_f16<DKQ, DV, ncols1, ncols2, false, false, false>;
++                static bool stock_limit_raised[GGML_CUDA_MAX_DEVICES] = {false};
++                if (!stock_limit_raised[id]) {
++                    CUDA_CHECK(cudaFuncSetAttribute(reinterpret_cast<fattn_kernel_ptr_t>(stock_kernel), cudaFuncAttributeMaxDynamicSharedMemorySize, nbytes_total_stock));
++                    stock_limit_raised[id] = true;
++                }
++                launch_fattn<DV, ncols1, ncols2>
++                    (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host,
++                     nstages_stock == 2, stock_kernel, nbytes_total_stock);
++            } else {
++                launch_fattn<DV, ncols1, ncols2>
++                    (ctx, dst, fattn_kernel, nwarps, nbytes_shared_total, nbatch_fa, true, true, true, use_sparse, warp_size_host,
++                     nstages == 2);
++            }
++            return;
++        }
++    }
+     if (logit_softcap == 0.0f) {
+         constexpr bool use_logit_softcap = false;
+ #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+@@ -2145,6 +2295,12 @@ DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(112, 112,  64)
+ DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(128, 128,  64)
+ DECL_FATTN_MMA_F16_CASE_ALL_NCOLS2(256, 256,  64)
+ 
++#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
++// Long-sequence variant only:
++extern DECL_FATTN_MMA_F16_CASE( 64,  64, 128, 1);
++extern DECL_FATTN_MMA_F16_CASE(128, 128, 128, 1);
++#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
++
+ extern DECL_FATTN_MMA_F16_CASE(512, 512,  4,  2);
+ extern DECL_FATTN_MMA_F16_CASE(512, 512,  8,  2);
+ extern DECL_FATTN_MMA_F16_CASE(512, 512, 16,  2);
+diff --git a/src/ggml-cuda/fattn.cu b/src/ggml-cuda/fattn.cu
+index 5fa3e530..9fb380e8 100644
+--- a/src/ggml-cuda/fattn.cu
++++ b/src/ggml-cuda/fattn.cu
+@@ -187,6 +187,15 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con
+         return;
+     }
+ 
++#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
++    if constexpr (ncols2 == 1 && DKQ == DV && (DKQ == 64 || DKQ == 128)) {
++        if (Q->ne[1] >= 1024 && ggml_cuda_fattn_mma_longseq_ncols(cc, DKQ) == 128 && ggml_cuda_fattn_mma_use_longseq(cc, dst)) {
++            ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 128, 1>(ctx, dst);
++            return;
++        }
++    }
++#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
++
+     ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 64/ncols2, ncols2>(ctx, dst);
+ }
+ 
+diff --git a/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_128-ncols2_1.cu b/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_128-ncols2_1.cu
+new file mode 100644
+index 00000000..b9bf54a1
+--- /dev/null
++++ b/src/ggml-cuda/template-instances/fattn-mma-f16-instance-ncols1_128-ncols2_1.cu
+@@ -0,0 +1,8 @@
++// Long-sequence FlashAttention variant (CUDA only), see ggml_cuda_fattn_mma_use_longseq in fattn-mma-f16.cuh.
++
++#include "../fattn-mma-f16.cuh"
++
++#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
++DECL_FATTN_MMA_F16_CASE(64, 64, 128, 1);
++DECL_FATTN_MMA_F16_CASE(128, 128, 128, 1);
++#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
+index 5e2db805..83136355 100644
+--- a/tests/test-backend-ops.cpp
++++ b/tests/test-backend-ops.cpp
+@@ -11710,6 +11710,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
+     test_cases.emplace_back(new test_flash_attn_ext(65, 67, 4, {1, 1}, 17, 75, false, false, 0, 1.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
+     test_cases.emplace_back(new test_flash_attn_ext(65, 67, 4, {1, 1}, 113, 75, false, false, 0, 1.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
+ 
++    // diffusion-shaped cases without a mask (CUDA long-sequence FA variant): KV lengths that are not a multiple
++    // of the KV tile (partial last tile), several Q tiles per head, and with/without GQA broadcast.
++    for (int hs : { 64, 128, }) {
++        for (int kv : { 260, 1000, 4100, }) {
++            for (int nb : { 64, 200, 1000, }) {
++                for (int nr2 : { 1, 4, }) {
++                    for (ggml_type type_KV : { GGML_TYPE_F16, GGML_TYPE_F32, }) {
++                        if (type_KV == GGML_TYPE_F32 && (nr2 != 1 || nb != 200)) continue;
++                        test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 4, {nr2, 1}, kv, nb, false, false, 0, 0, GGML_PREC_F32, type_KV, type_KV));
++                    }
++                }
++            }
++        }
++    }
++
+     // mixed quant and Q1_0 test cases
+     test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0));
+     test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16));
diff --git a/scripts/unsloth/ggml-patches/0003-ggml-cuda-h3-vae-epilogue-fusion.patch b/scripts/unsloth/ggml-patches/0003-ggml-cuda-h3-vae-epilogue-fusion.patch
new file mode 100644
index 0000000000..0d086f6214
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0003-ggml-cuda-h3-vae-epilogue-fusion.patch
@@ -0,0 +1,350 @@
+From 54d3121a3e39ddf79c025a87271c9e349fb76b5d Mon Sep 17 00:00:00 2001
+From: Daniel Han <danielhanchen@gmail.com>
+Date: Thu, 1 Oct 2026 05:46:18 -0700
+Subject: [PATCH] ggml-cuda: fuse bias epilogues into F16 cuBLAS matmuls,
+ narrow blocks for short-row RMS norm
+
+An F16 matmul on the cuBLAS path writes an F16 temporary, converts it to F32, and the
+graph then adds a bias (and in transformer blocks multiplies by a per-channel scale and
+adds the residual, or applies SWIGLU) in separate passes over the F32 result. When those
+nodes directly follow the matmul and have no other users, they now run inside the F16 -> F32
+conversion: MUL_MAT+ADD, MUL_MAT+ADD+MUL+ADD and MUL_MAT+ADD+GLU(SWIGLU). Every element
+goes through the same rounded operations in the same order (explicit _rn intrinsics stop
+nvcc from contracting them into FMAs), so results are bit-identical.
+GGML_CUDA_CUBLAS_EPILOGUE_FUSION=0 disables it.
+
+RMS norm over rows of at most 128 columns (per-head q/k norms) launched 256-thread blocks
+in which most threads had no element. It now uses the next multiple of 32 threads: each
+thread still owns the same element and each warp the same partial sum, and the wider block
+only added zero partials, so the result is unchanged. GGML_CUDA_NORM_SMALL_ROWS=0 restores
+the 256-thread launch.
+---
+ src/ggml-cuda/ggml-cuda.cu | 200 ++++++++++++++++++++++++++++++++++++-
+ src/ggml-cuda/norm.cu      |  63 ++++++++++++
+ 2 files changed, 262 insertions(+), 1 deletion(-)
+
+[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is
+ the whole patch as one diff, regenerated from the port.]
+---
+diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu
+index 8a295c19..74f3373a 100644
+--- a/src/ggml-cuda/ggml-cuda.cu
++++ b/src/ggml-cuda/ggml-cuda.cu
+@@ -2061,8 +2061,73 @@ struct batched_mul_mat_traits<GGML_TYPE_F16> {
+     static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_fp16_nc_cuda(src_type); }
+ };
+ 
++// Elementwise ops that directly follow an F16 cuBLAS matmul, applied while converting its F16
++// output to F32 instead of in separate passes over the F32 result. Each element sees the same
++// rounded operations in the same order as the unfused ADD / MUL / ADD or ADD / SWIGLU nodes
++// (explicit _rn intrinsics keep nvcc from contracting them into FMAs), so results are bit-identical.
++enum ggml_cuda_cublas_epilogue_mode {
++    GGML_CUDA_CUBLAS_EPILOGUE_BIAS,               // dst = mm + bias
++    GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SCALE_RESIDUAL, // dst = residual + (mm + bias) * scale
++    GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SWIGLU,         // dst = silu(mm[:n] + bias[:n]) * (mm[n:] + bias[n:])
++};
++
++struct ggml_cuda_cublas_epilogue {
++    ggml_cuda_cublas_epilogue_mode mode;
++    const float * bias;
++    const float * scale;
++    const float * residual;
++    float *       dst;
++    int64_t       ne0_out;
++    int64_t       n_out;
++};
++
++template <ggml_cuda_cublas_epilogue_mode mode>
++static __global__ void k_cublas_f16_epilogue(const half * __restrict__ mm, const float * __restrict__ bias,
++                                             const float * __restrict__ scale, const float * residual, float * dst,
++                                             const int64_t ne0_out, const int64_t n_out) {
++    const int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x;
++    if (i >= n_out) {
++        return;
++    }
++    const int64_t row = i / ne0_out;
++    const int64_t col = i - row*ne0_out;
++    if constexpr (mode == GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SWIGLU) {
++        const int64_t j = row*2*ne0_out + col;
++        const float   x = __fadd_rn(__half2float(mm[j]), bias[col]);
++        const float   g = __fadd_rn(__half2float(mm[j + ne0_out]), bias[col + ne0_out]);
++        dst[i] = __fmul_rn(ggml_cuda_op_silu_single(x), g);
++    } else {
++        float v = __fadd_rn(__half2float(mm[i]), bias[col]);
++        if constexpr (mode == GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SCALE_RESIDUAL) {
++            v = __fadd_rn(residual[i], __fmul_rn(v, scale[col]));
++        }
++        dst[i] = v;
++    }
++}
++
++static void ggml_cuda_cublas_f16_epilogue(const half * mm, const ggml_cuda_cublas_epilogue & epi, cudaStream_t stream) {
++    const int     block = 256;
++    const int64_t grid  = (epi.n_out + block - 1) / block;
++    switch (epi.mode) {
++        case GGML_CUDA_CUBLAS_EPILOGUE_BIAS:
++            k_cublas_f16_epilogue<GGML_CUDA_CUBLAS_EPILOGUE_BIAS><<<grid, block, 0, stream>>>(
++                mm, epi.bias, epi.scale, epi.residual, epi.dst, epi.ne0_out, epi.n_out);
++            break;
++        case GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SCALE_RESIDUAL:
++            k_cublas_f16_epilogue<GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SCALE_RESIDUAL><<<grid, block, 0, stream>>>(
++                mm, epi.bias, epi.scale, epi.residual, epi.dst, epi.ne0_out, epi.n_out);
++            break;
++        case GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SWIGLU:
++            k_cublas_f16_epilogue<GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SWIGLU><<<grid, block, 0, stream>>>(
++                mm, epi.bias, epi.scale, epi.residual, epi.dst, epi.ne0_out, epi.n_out);
++            break;
++    }
++    CUDA_CHECK(cudaGetLastError());
++}
++
+ template<ggml_type compute_type>
+-static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
++static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
++                                          const ggml_cuda_cublas_epilogue * epilogue = nullptr) {
+     using traits = batched_mul_mat_traits<compute_type>;
+     using cuda_t = typename traits::cuda_type;
+ 
+@@ -2270,8 +2335,17 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const
+ 
+     // Convert output back to F32 if needed
+     if (cu_data_type != CUDA_R_32F) {
++        if constexpr (compute_type == GGML_TYPE_F16) {
++            if (epilogue != nullptr) {
++                ggml_cuda_cublas_f16_epilogue((const half *) dst_temp.get(), *epilogue, main_stream);
++                return;
++            }
++        }
++        GGML_ASSERT(epilogue == nullptr);
+         const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(traits::ggml_type_val);
+         to_fp32_cuda(dst_temp.get(), dst_ddf, ne_dst, main_stream);
++    } else {
++        GGML_ASSERT(epilogue == nullptr);
+     }
+ }
+ 
+@@ -4336,6 +4410,123 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph *                cgraph,
+ }
+ 
+ // try and fuse nodes and return the number of nodes to skip
++static bool ggml_cuda_cublas_epilogue_enabled() {
++    static const bool enabled = [] {
++        const char * e = getenv("GGML_CUDA_CUBLAS_EPILOGUE_FUSION");
++        return e == nullptr || atoi(e) != 0;
++    }();
++    return enabled;
++}
++
++// True when ggml_cuda_mul_mat sends this F16 matmul to cuBLAS with F16 compute and an F16
++// temporary output (the path the epilogue fusion replaces the F16 -> F32 conversion of).
++static bool ggml_cuda_mul_mat_uses_cublas_f16_temp(const ggml_backend_cuda_context & ctx, const ggml_tensor * mm) {
++    const ggml_tensor * src0 = mm->src[0];
++    const ggml_tensor * src1 = mm->src[1];
++    if (src0->type != GGML_TYPE_F16 || src1->type != GGML_TYPE_F32 || mm->type != GGML_TYPE_F32 ||
++        !ggml_is_contiguous(mm) || ggml_get_op_params_i32(mm, 1) == GGML_HINT_SRC0_IS_HADAMARD ||
++        mm->op_params[0] == GGML_PREC_F32 || getenv("GGML_CUDA_CUBLAS_COMPUTE_TYPE") != nullptr) {
++        return false;
++    }
++    const int cc        = ggml_cuda_info().devices[ctx.device].cc;
++    const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size;
++    if (!fast_fp16_hardware_available(cc) || cc == GGML_CUDA_CC_VOLTA || GGML_CUDA_CC_IS_RDNA4(cc) || GGML_CUDA_CC_IS_CDNA(cc)) {
++        return false;
++    }
++    const bool bad_padding_clear = ggml_backend_buffer_get_usage(src0->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE
++        && ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) && src0->view_src;
++    if (bad_padding_clear) {
++        return true;
++    }
++    const int64_t ne11 = src1->ne[1];
++    return !ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11) &&
++           !ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false);
++}
++
++static const ggml_tensor * ggml_cuda_other_src(const ggml_tensor * node, const ggml_tensor * operand) {
++    if (node->src[0] == operand) {
++        return node->src[1];
++    }
++    if (node->src[1] == operand) {
++        return node->src[0];
++    }
++    return nullptr;
++}
++
++// Like ggml_node_has_n_uses(cgraph, idx, 1), but also accepts an in-place node whose view
++// source is the (single-use) node before it, e.g. the bias ADD that ggml_add_inplace builds.
++static bool ggml_cuda_single_use_intermediate(const ggml_cgraph * cgraph, int idx, const ggml_tensor * prev) {
++    const ggml_tensor * node = cgraph->nodes[idx];
++    if (ggml_node_get_use_count(cgraph, idx) != 1 || (node->flags & GGML_TENSOR_FLAG_OUTPUT)) {
++        return false;
++    }
++    return node->view_src == nullptr || node->view_src == prev;
++}
++
++static bool ggml_cuda_is_row_vector(const ggml_tensor * t, int64_t ne0) {
++    return t != nullptr && t->type == GGML_TYPE_F32 && ggml_is_contiguous(t) && t->ne[0] == ne0 && ggml_nelements(t) == ne0;
++}
++
++// MUL_MAT (F16 weights, cuBLAS) followed by a row bias ADD, optionally then MUL by a row scale and
++// ADD of a residual, or SWIGLU. Returns the number of extra nodes consumed, 0 if not fused.
++static int ggml_cuda_try_fuse_cublas_epilogue(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i) {
++    if (!ggml_cuda_cublas_epilogue_enabled() || i + 1 >= cgraph->n_nodes) {
++        return 0;
++    }
++    ggml_tensor * mm  = cgraph->nodes[i];
++    ggml_tensor * add = cgraph->nodes[i + 1];
++    if (mm->op != GGML_OP_MUL_MAT || add->op != GGML_OP_ADD || (add->flags & GGML_TENSOR_FLAG_COMPUTE) == 0 ||
++        !ggml_node_has_n_uses(cgraph, i, 1) || !ggml_are_same_shape(mm, add) || add->type != GGML_TYPE_F32 ||
++        !ggml_is_contiguous(add)) {
++        return 0;
++    }
++    const ggml_tensor * bias = ggml_cuda_other_src(add, mm);
++    if (!ggml_cuda_is_row_vector(bias, mm->ne[0]) || !ggml_cuda_mul_mat_uses_cublas_f16_temp(ctx, mm)) {
++        return 0;
++    }
++
++    ggml_cuda_cublas_epilogue epi = {};
++    epi.mode    = GGML_CUDA_CUBLAS_EPILOGUE_BIAS;
++    epi.bias    = (const float *) bias->data;
++    epi.dst     = (float *) add->data;
++    epi.ne0_out = add->ne[0];
++    epi.n_out   = ggml_nelements(add);
++    int consumed = 1;
++
++    if (i + 2 < cgraph->n_nodes && ggml_cuda_single_use_intermediate(cgraph, i + 1, mm)) {
++        ggml_tensor * mul = cgraph->nodes[i + 2];
++        ggml_tensor * res = i + 3 < cgraph->n_nodes ? cgraph->nodes[i + 3] : nullptr;
++        const ggml_tensor * glu = cgraph->nodes[i + 2];
++        if (res != nullptr && mul->op == GGML_OP_MUL && res->op == GGML_OP_ADD && (mul->flags & GGML_TENSOR_FLAG_COMPUTE) &&
++            (res->flags & GGML_TENSOR_FLAG_COMPUTE) && ggml_node_has_n_uses(cgraph, i + 2, 1) &&
++            ggml_are_same_shape(mul, add) && ggml_are_same_shape(res, add) && mul->type == GGML_TYPE_F32 &&
++            res->type == GGML_TYPE_F32 && ggml_is_contiguous(res)) {
++            const ggml_tensor * scale    = ggml_cuda_other_src(mul, add);
++            const ggml_tensor * residual = ggml_cuda_other_src(res, mul);
++            if (mul->src[0] == add && ggml_cuda_is_row_vector(scale, add->ne[0]) && residual != nullptr &&
++                residual->type == GGML_TYPE_F32 && ggml_is_contiguous(residual) && ggml_are_same_shape(residual, res)) {
++                epi.mode     = GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SCALE_RESIDUAL;
++                epi.scale    = (const float *) scale->data;
++                epi.residual = (const float *) residual->data;
++                epi.dst      = (float *) res->data;
++                consumed     = 3;
++            }
++        } else if (glu->op == GGML_OP_GLU && (glu->flags & GGML_TENSOR_FLAG_COMPUTE) && glu->src[0] == add &&
++                   glu->src[1] == nullptr && ggml_get_glu_op(glu) == GGML_GLU_OP_SWIGLU &&
++                   ggml_get_op_params_i32(glu, 1) == 0 && glu->type == GGML_TYPE_F32 && ggml_is_contiguous(glu) &&
++                   add->ne[0] % 2 == 0 && glu->ne[0] * 2 == add->ne[0] && ggml_nrows(glu) == ggml_nrows(add)) {
++            epi.mode    = GGML_CUDA_CUBLAS_EPILOGUE_BIAS_SWIGLU;
++            epi.dst     = (float *) glu->data;
++            epi.ne0_out = glu->ne[0];
++            epi.n_out   = ggml_nelements(glu);
++            consumed    = 2;
++        }
++    }
++
++    ggml_cuda_mul_mat_cublas_impl<GGML_TYPE_F16>(ctx, mm->src[0], mm->src[1], mm, &epi);
++    return consumed;
++}
++
+ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) {
+ 
+     if (ggml_cuda_fusion_disabled()) {
+@@ -5083,6 +5274,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
+         return 2;
+     }
+ 
++    if (node->op == GGML_OP_MUL_MAT) {
++        const int consumed = ggml_cuda_try_fuse_cublas_epilogue(*cuda_ctx, cgraph, i);
++        if (consumed > 0) {
++            return consumed;
++        }
++    }
++
+     if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
+         ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
+         return 2;
+diff --git a/src/ggml-cuda/norm.cu b/src/ggml-cuda/norm.cu
+index d804b933..854d47b7 100644
+--- a/src/ggml-cuda/norm.cu
++++ b/src/ggml-cuda/norm.cu
+@@ -332,12 +332,70 @@ static void group_norm_f32_cuda(
+     }
+ }
+ 
++// Rows of at most 128 columns (per-head q/k norms) leave most of a 256-thread block idle. A block
++// of the next multiple of 32 gives every thread the same element and every warp the same partial
++// sum; the 256-thread reduction only adds zero partials on top, so the result is bit-identical.
++// GGML_CUDA_NORM_SMALL_ROWS=0 restores the 256-thread launch.
++static bool rms_norm_small_rows_enabled() {
++#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
++    return false;
++#else
++    static const bool enabled = [] {
++        const char * e = getenv("GGML_CUDA_NORM_SMALL_ROWS");
++        return e == nullptr || atoi(e) != 0;
++    }();
++    return enabled && WARP_SIZE == 32;
++#endif
++}
++
++template <int block_size, bool do_multiply, bool do_scale>
++static void rms_norm_f32_launch_small(
++        const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, const dim3 blocks_num,
++        const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps,
++        const float * mul, const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample,
++        const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed,
++        cudaStream_t stream, const float scale_out) {
++    const dim3 block_dims(block_size, 1, 1);
++    const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, block_size > WARP_SIZE ? 32 * sizeof(float) : 0, stream};
++    ggml_cuda_kernel_launch(rms_norm_f32<block_size, do_multiply, false, do_scale>, launch_params,
++        x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, mul_stride_row, mul_stride_channel,
++        mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed,
++        nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), scale_out);
++}
++
++template <bool do_multiply, bool do_scale = false>
++static bool rms_norm_f32_try_small(
++        const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, const dim3 blocks_num,
++        const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps,
++        const float * mul, const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample,
++        const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed,
++        cudaStream_t stream, const float scale_out = 1.0f) {
++    if (ncols > 128 || !rms_norm_small_rows_enabled()) {
++        return false;
++    }
++    if (ncols <= 32) {
++        rms_norm_f32_launch_small<32, do_multiply, do_scale>(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps, mul,
++            mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream, scale_out);
++    } else if (ncols <= 64) {
++        rms_norm_f32_launch_small<64, do_multiply, do_scale>(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps, mul,
++            mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream, scale_out);
++    } else {
++        rms_norm_f32_launch_small<128, do_multiply, do_scale>(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps, mul,
++            mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, stream, scale_out);
++    }
++    return true;
++}
++
+ template <bool do_scale = false>
+ static void rms_norm_f32_cuda(
+         const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,
+         const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream,
+         const float scale_out = 1.0f) {
+     const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX));
++    if (rms_norm_f32_try_small<false, do_scale>(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps,
++            nullptr, 0, 0, 0, make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), make_uint3(0, 0, 0), stream, scale_out)) {
++        return;
++    }
+     if (ncols < 1024) {
+         const dim3 block_dims(256, 1, 1);
+         const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
+@@ -393,6 +451,11 @@ static void rms_norm_mul_f32_cuda(const float *  x,
+         const uint3 mul_nrows_packed     = init_fastdiv_values(mul_nrows);
+         const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels);
+         const uint3 mul_nsamples_packed  = init_fastdiv_values(mul_nsamples);
++        if (rms_norm_f32_try_small<true>(x, dst, ncols, nchannels, nsamples, blocks_num, stride_row, stride_channel, stride_sample, eps,
++                mul, mul_stride_row, mul_stride_channel, mul_stride_sample, mul_ncols_packed, mul_nrows_packed,
++                mul_nchannels_packed, mul_nsamples_packed, stream)) {
++            return;
++        }
+         if (ncols < 1024) {
+             const dim3 block_dims(256, 1, 1);
+             const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params{blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float): 0, stream};
diff --git a/scripts/unsloth/ggml-patches/0004-ggml-h3-dit-fused-ops.patch b/scripts/unsloth/ggml-patches/0004-ggml-h3-dit-fused-ops.patch
new file mode 100644
index 0000000000..386d09a04b
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0004-ggml-h3-dit-fused-ops.patch
@@ -0,0 +1,1020 @@
+From cc2ac8fad7ecb73b19cb7971c874883903393a96 Mon Sep 17 00:00:00 2001
+From: Daniel Han <danielhanchen@gmail.com>
+Date: Thu, 1 Oct 2026 06:48:01 -0700
+Subject: [PATCH] ggml : fused value-preserving ops for adaLN DiT blocks
+
+Three ops that replace long chains of elementwise ops, copies and concats in
+diffusion transformers such as MiniMax-H3 while producing the same bits as the
+chains they replace (every product, sum, scale and conversion is rounded
+separately, never contracted into an FMA):
+
+- ggml_rope_pe_permute: rotary embedding with a precomputed [cos, -sin; sin,
+  cos] table on the first n_rot channels (non-interleaved pairing), pass-through
+  for the rest, written in the head-major [d, tokens, heads] layout attention
+  reads, with an optional scale and F16 cast. Reads a strided view, so the
+  q/k/v chunks of a fused qkv projection need no copies.
+- ggml_modulate_rows / ggml_gated_add_rows: (x + x*scale) + shift and
+  base + u*gate on a row range, written in place into an accumulator so a
+  sequence split into segments with different modulation vectors needs no
+  slice copies and no concat. Optional output / input scale.
+- ggml_swiglu_scaled: silu(s_in*a) * (s_in*b) scaled by s_out, i.e. a Linear
+  output scale, swiglu and the next Linear input scale in one pass.
+
+CPU and CUDA implementations; other backends report the ops unsupported so
+callers keep their unfused graphs. test-backend-ops covers strided inputs,
+partial and full rotation, F32/F16 outputs, chained segments and the scales.
+---
+ include/ggml.h                  |  62 ++++++++++++
+ src/ggml-backend-meta.cpp       |   3 +-
+ src/ggml-cpu/ggml-cpu.c         |  15 +++
+ src/ggml-cpu/ops.cpp            | 154 +++++++++++++++++++++++++++++
+ src/ggml-cpu/ops.h              |   3 +
+ src/ggml-cuda/ggml-cuda.cu      |  17 ++++
+ src/ggml-cuda/modulate-rows.cu  | 168 ++++++++++++++++++++++++++++++++
+ src/ggml-cuda/modulate-rows.cuh |   7 ++
+ src/ggml-cuda/rope-pe.cu        | 102 +++++++++++++++++++
+ src/ggml-cuda/rope-pe.cuh       |   5 +
+ src/ggml.c                      | 129 +++++++++++++++++++++++-
+ tests/test-backend-ops.cpp      | 139 ++++++++++++++++++++++++++
+ 12 files changed, 801 insertions(+), 3 deletions(-)
+ create mode 100644 src/ggml-cuda/modulate-rows.cu
+ create mode 100644 src/ggml-cuda/modulate-rows.cuh
+ create mode 100644 src/ggml-cuda/rope-pe.cu
+ create mode 100644 src/ggml-cuda/rope-pe.cuh
+
+[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is
+ the whole patch as one diff, regenerated from the port.]
+---
+diff --git a/include/ggml.h b/include/ggml.h
+index 49a6751d..bbafe175 100644
+--- a/include/ggml.h
++++ b/include/ggml.h
+@@ -607,6 +607,9 @@ extern "C" {
+         GGML_OP_SAGE_ATTN,
+         GGML_OP_SOL_ATTN,
+         GGML_OP_ROPE_APPLY,
++        GGML_OP_ROPE_PE_PERMUTE,
++        GGML_OP_MODULATE_ROWS,
++        GGML_OP_SWIGLU_SCALED,
+ 
+         GGML_OP_COUNT,
+     };
+@@ -2556,6 +2559,65 @@ extern "C" {
+         float scale,
+         float tau);
+ 
++    // Rotary embedding with a precomputed cos/sin table, fused with the head-major relayout
++    // attention kernels consume, an optional scale and an optional cast to F16.
++    // a:      F32 [head_dim, n_head, n_tokens, n_batch], rows contiguous (other strides free)
++    // pe:     F32 contiguous [2, 2, n_rot/2, n_tokens], pe[c, r, j, t] = M[r][c] of the 2x2 rotation
++    //         for pair j of token t (non-interleaved pairing j <-> j + n_rot/2); NULL when n_rot == 0
++    // result: `type` (F32 or F16) contiguous [head_dim, n_tokens, n_head, n_batch]
++    //         i <  n_rot: y = a[j]*pe[0, r, j, t] + a[j + n_rot/2]*pe[1, r, j, t] (r = i / (n_rot/2), j = i % (n_rot/2))
++    //         i >= n_rot: y = a[i]
++    //         y = scale*y when scale != 1, then y is converted to `type`.
++    // Each product, the sum, the scale and the conversion are rounded separately, so the result
++    // matches the equivalent chain of ggml_mul/ggml_add/ggml_concat/ggml_scale/ggml_cpy bit for bit.
++    GGML_API struct ggml_tensor * ggml_rope_pe_permute(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * a,
++            struct ggml_tensor  * pe,
++            int                   n_rot,
++            float                 scale,
++            enum ggml_type        type);
++
++    // Row-range modulation for adaLN-style blocks whose sequence is split into segments that use
++    // different modulation vectors. Writes rows [row0, row1) along dim 1 (all of dims 2 and 3) of the
++    // result and leaves every other row as it is in `acc`: chain one call per segment, passing the
++    // previous result as `acc` (written in place); the first call passes acc = NULL and gets a new
++    // tensor shaped like x / base. x, base, u: F32 with contiguous rows; scale, shift, gate: F32
++    // with ne[0] == x->ne[0] elements, broadcast over rows.
++    //   ggml_modulate_rows:  y = (x + x*scale) + shift; then y = out_scale*y when out_scale != 1
++    //   ggml_gated_add_rows: w = u_scale*u when u_scale != 1 (else u); y = base + w*gate
++    // Each product, sum and scale is rounded separately, so the result equals the slice +
++    // ggml_mul/ggml_add/ggml_scale + ggml_concat chain bit for bit.
++    GGML_API struct ggml_tensor * ggml_modulate_rows(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * acc,
++            struct ggml_tensor  * x,
++            struct ggml_tensor  * scale,
++            struct ggml_tensor  * shift,
++            int64_t               row0,
++            int64_t               row1,
++            float                 out_scale);
++
++    GGML_API struct ggml_tensor * ggml_gated_add_rows(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * acc,
++            struct ggml_tensor  * base,
++            struct ggml_tensor  * u,
++            struct ggml_tensor  * gate,
++            int64_t               row0,
++            int64_t               row1,
++            float                 u_scale);
++
++    // swiglu on two separately scaled halves, then scaled: with a' = in_scale*a, b' = in_scale*b,
++    // y = out_scale*(silu(a')*b') (each scale skipped when it is 1). a, b: F32, same shape, rows
++    // contiguous. Matches ggml_scale -> ggml_swiglu_split -> ggml_scale bit for bit.
++    GGML_API struct ggml_tensor * ggml_swiglu_scaled(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * a,
++            struct ggml_tensor  * b,
++            float                 in_scale,
++            float                 out_scale);
++
+     GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(
+             struct ggml_tensor * a,
+             enum ggml_prec       prec),
+diff --git a/src/ggml-backend-meta.cpp b/src/ggml-backend-meta.cpp
+index fb447e97..899c8ad4 100644
+--- a/src/ggml-backend-meta.cpp
++++ b/src/ggml-backend-meta.cpp
+@@ -153,7 +153,8 @@ static ggml_backend_buffer_type_t ggml_backend_meta_device_get_host_buffer_type(
+ 
+ static bool ggml_backend_meta_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) {
+     GGML_ASSERT(ggml_backend_dev_is_meta(dev));
+-    if (op->op == GGML_OP_SAGE_ATTN || op->op == GGML_OP_SOL_ATTN) {
++    if (op->op == GGML_OP_SAGE_ATTN || op->op == GGML_OP_SOL_ATTN || op->op == GGML_OP_ROPE_PE_PERMUTE ||
++        op->op == GGML_OP_MODULATE_ROWS || op->op == GGML_OP_SWIGLU_SCALED) {
+         return false;
+     }
+     const ggml_backend_meta_device_context * meta_dev_ctx = (const ggml_backend_meta_device_context *) dev->context;
+diff --git a/src/ggml-cpu/ggml-cpu.c b/src/ggml-cpu/ggml-cpu.c
+index fd55589f..30b8cbb5 100644
+--- a/src/ggml-cpu/ggml-cpu.c
++++ b/src/ggml-cpu/ggml-cpu.c
+@@ -2220,6 +2220,18 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
+             {
+                 ggml_compute_forward_glu(params, tensor);
+             } break;
++        case GGML_OP_ROPE_PE_PERMUTE:
++            {
++                ggml_compute_forward_rope_pe_permute(params, tensor);
++            } break;
++        case GGML_OP_MODULATE_ROWS:
++            {
++                ggml_compute_forward_modulate_rows(params, tensor);
++            } break;
++        case GGML_OP_SWIGLU_SCALED:
++            {
++                ggml_compute_forward_swiglu_scaled(params, tensor);
++            } break;
+         case GGML_OP_GET_REL_POS:
+             {
+                 ggml_compute_forward_get_rel_pos(params, tensor);
+@@ -2554,6 +2566,9 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
+         case GGML_OP_ROPE_BACK:
+         case GGML_OP_ROPE_APPLY:
+         case GGML_OP_ADD_REL_POS:
++        case GGML_OP_ROPE_PE_PERMUTE:
++        case GGML_OP_MODULATE_ROWS:
++        case GGML_OP_SWIGLU_SCALED:
+             {
+                 n_tasks = n_threads;
+             } break;
+diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp
+index e8ef14f2..94333d30 100644
+--- a/src/ggml-cpu/ops.cpp
++++ b/src/ggml-cpu/ops.cpp
+@@ -12,6 +12,7 @@
+ #include <algorithm>
+ #include <cfloat>
+ #include <cmath>
++#include <vector>
+ 
+ struct ggml_fp8_e4m3_cpu {
+     uint8_t value;
+@@ -12686,3 +12687,156 @@ void ggml_compute_forward_quantize_i8_convrot(const ggml_compute_params * params
+         memset(dst_data + rows * dst->nb[1], 0, (size_t)(rows_padded - rows) * dst->nb[1]);
+     }
+ }
++
++// ggml_compute_forward_rope_pe_permute
++
++void ggml_compute_forward_rope_pe_permute(
++        const ggml_compute_params * params,
++        ggml_tensor * dst) {
++    const ggml_tensor * src0 = dst->src[0];
++    const ggml_tensor * pe   = dst->src[1];
++
++    GGML_ASSERT(src0->type == GGML_TYPE_F32);
++    GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
++
++    const int   n_rot = ggml_get_op_params_i32(dst, 0);
++    const float scale = ggml_get_op_params_f32(dst, 1);
++    const int   half  = n_rot / 2;
++
++    const int64_t D = src0->ne[0];
++    const int64_t H = src0->ne[1];
++    const int64_t L = src0->ne[2];
++    const int64_t N = src0->ne[3];
++
++    const int64_t nrows = H * L * N;
++    const int64_t dr    = (nrows + params->nth - 1) / params->nth;
++    const int64_t ir0   = dr * params->ith;
++    const int64_t ir1   = MIN(ir0 + dr, nrows);
++
++    // Same rounding points as the unfused graph: each product, the sum, the scale and the
++    // conversion are separate passes, so no contraction into fused multiply-adds can happen.
++    std::vector<float> p0(n_rot), p1(n_rot), y(D);
++
++    for (int64_t ir = ir0; ir < ir1; ++ir) {
++        const int64_t h = ir % H;
++        const int64_t l = (ir / H) % L;
++        const int64_t n = ir / (H * L);
++
++        const float * x = (const float *) ((const char *) src0->data + h * src0->nb[1] + l * src0->nb[2] + n * src0->nb[3]);
++
++        if (n_rot > 0) {
++            const float * t = (const float *) pe->data + l * (int64_t) half * 4;
++            for (int i = 0; i < n_rot; ++i) {
++                const int r = i / half;
++                const int j = i - r * half;
++                p0[i]       = x[j] * t[4 * j + 2 * r + 0];
++            }
++            for (int i = 0; i < n_rot; ++i) {
++                const int r = i / half;
++                const int j = i - r * half;
++                p1[i]       = x[j + half] * t[4 * j + 2 * r + 1];
++            }
++            ggml_vec_add_f32(n_rot, y.data(), p0.data(), p1.data());
++        }
++        if (D > n_rot) {
++            memcpy(y.data() + n_rot, x + n_rot, (D - n_rot) * sizeof(float));
++        }
++        if (scale != 1.0f) {
++            ggml_vec_scale_f32((int) D, y.data(), scale);
++        }
++
++        char * out = (char *) dst->data + l * dst->nb[1] + h * dst->nb[2] + n * dst->nb[3];
++        if (dst->type == GGML_TYPE_F32) {
++            memcpy(out, y.data(), D * sizeof(float));
++        } else {
++            ggml_cpu_fp32_to_fp16(y.data(), (ggml_fp16_t *) out, D);
++        }
++    }
++}
++
++// ggml_compute_forward_modulate_rows
++
++void ggml_compute_forward_modulate_rows(
++        const ggml_compute_params * params,
++        ggml_tensor * dst) {
++    const ggml_tensor * a = dst->src[0];
++    const ggml_tensor * b = dst->src[1];
++    const ggml_tensor * c = dst->src[2];
++
++    const int     mode = ggml_get_op_params_i32(dst, 0);
++    const int64_t row0 = ggml_get_op_params_i32(dst, 1);
++    const int64_t row1 = ggml_get_op_params_i32(dst, 2);
++    const float   s    = ggml_get_op_params_f32(dst, 3);
++
++    const int64_t nc    = a->ne[0];
++    const int64_t nr1   = row1 - row0;
++    const int64_t nrows = nr1 * a->ne[2] * a->ne[3];
++    const int64_t dr    = (nrows + params->nth - 1) / params->nth;
++    const int64_t ir0   = dr * params->ith;
++    const int64_t ir1   = MIN(ir0 + dr, nrows);
++
++    // separate passes keep every product, sum and scale individually rounded, as in the unfused graph
++    std::vector<float> t(nc), w(nc);
++    for (int64_t ir = ir0; ir < ir1; ++ir) {
++        const int64_t i1 = row0 + ir % nr1;
++        const int64_t i2 = (ir / nr1) % a->ne[2];
++        const int64_t i3 = ir / (nr1 * a->ne[2]);
++
++        const float * x = (const float *) ((const char *) a->data + i1 * a->nb[1] + i2 * a->nb[2] + i3 * a->nb[3]);
++        float * y       = (float *) ((char *) dst->data + i1 * dst->nb[1] + i2 * dst->nb[2] + i3 * dst->nb[3]);
++
++        if (mode == 0) {
++            ggml_vec_mul_f32((int) nc, t.data(), x, (const float *) b->data);
++            ggml_vec_add_f32((int) nc, w.data(), x, t.data());
++            ggml_vec_add_f32((int) nc, y, w.data(), (const float *) c->data);
++            if (s != 1.0f) {
++                ggml_vec_scale_f32((int) nc, y, s);
++            }
++        } else {
++            const float * u = (const float *) ((const char *) b->data + i1 * b->nb[1] + i2 * b->nb[2] + i3 * b->nb[3]);
++            memcpy(w.data(), u, nc * sizeof(float));
++            if (s != 1.0f) {
++                ggml_vec_scale_f32((int) nc, w.data(), s);
++            }
++            ggml_vec_mul_f32((int) nc, t.data(), w.data(), (const float *) c->data);
++            ggml_vec_add_f32((int) nc, y, x, t.data());
++        }
++    }
++}
++
++// ggml_compute_forward_swiglu_scaled
++
++void ggml_compute_forward_swiglu_scaled(
++        const ggml_compute_params * params,
++        ggml_tensor * dst) {
++    const ggml_tensor * a = dst->src[0];
++    const ggml_tensor * b = dst->src[1];
++
++    const float in_scale  = ggml_get_op_params_f32(dst, 0);
++    const float out_scale = ggml_get_op_params_f32(dst, 1);
++
++    const int64_t nc    = a->ne[0];
++    const int64_t nrows = ggml_nrows(a);
++    const int64_t dr    = (nrows + params->nth - 1) / params->nth;
++    const int64_t ir0   = dr * params->ith;
++    const int64_t ir1   = MIN(ir0 + dr, nrows);
++
++    std::vector<float> xa(nc), xb(nc);
++    for (int64_t ir = ir0; ir < ir1; ++ir) {
++        const int64_t i1 = ir % a->ne[1];
++        const int64_t i2 = (ir / a->ne[1]) % a->ne[2];
++        const int64_t i3 = ir / (a->ne[1] * a->ne[2]);
++
++        memcpy(xa.data(), (const char *) a->data + i1 * a->nb[1] + i2 * a->nb[2] + i3 * a->nb[3], nc * sizeof(float));
++        memcpy(xb.data(), (const char *) b->data + i1 * b->nb[1] + i2 * b->nb[2] + i3 * b->nb[3], nc * sizeof(float));
++        if (in_scale != 1.0f) {
++            ggml_vec_scale_f32((int) nc, xa.data(), in_scale);
++            ggml_vec_scale_f32((int) nc, xb.data(), in_scale);
++        }
++        float * y = (float *) ((char *) dst->data + i1 * dst->nb[1] + i2 * dst->nb[2] + i3 * dst->nb[3]);
++        ggml_vec_swiglu_f32((int) nc, y, xa.data(), xb.data());
++        if (out_scale != 1.0f) {
++            ggml_vec_scale_f32((int) nc, y, out_scale);
++        }
++    }
++}
+diff --git a/src/ggml-cpu/ops.h b/src/ggml-cpu/ops.h
+index 19d7075e..d731a958 100644
+--- a/src/ggml-cpu/ops.h
++++ b/src/ggml-cpu/ops.h
+@@ -98,6 +98,9 @@ void ggml_compute_forward_win_part(const struct ggml_compute_params * params, st
+ void ggml_compute_forward_win_unpart(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_unary(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_glu(const struct ggml_compute_params * params, struct ggml_tensor * dst);
++void ggml_compute_forward_rope_pe_permute(const struct ggml_compute_params * params, struct ggml_tensor * dst);
++void ggml_compute_forward_modulate_rows(const struct ggml_compute_params * params, struct ggml_tensor * dst);
++void ggml_compute_forward_swiglu_scaled(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_get_rel_pos(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_add_rel_pos(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+ void ggml_compute_forward_rwkv_wkv6(const struct ggml_compute_params * params, struct ggml_tensor * dst);
+diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu
+index cd0cee80..5c799ccd 100644
+--- a/src/ggml-cuda/ggml-cuda.cu
++++ b/src/ggml-cuda/ggml-cuda.cu
+@@ -47,6 +47,8 @@
+ #include "ggml-cuda/rope.cuh"
+ #include "ggml-cuda/roll.cuh"
+ #include "ggml-cuda/scale.cuh"
++#include "ggml-cuda/rope-pe.cuh"
++#include "ggml-cuda/modulate-rows.cuh"
+ #include "ggml-cuda/snake.cuh"
+ #include "ggml-cuda/softcap.cuh"
+ #include "ggml-cuda/softmax.cuh"
+@@ -3253,6 +3255,15 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
+         case GGML_OP_SOL_ATTN:
+             ggml_cuda_sol_attn(ctx, dst);
+             break;
++        case GGML_OP_ROPE_PE_PERMUTE:
++            ggml_cuda_op_rope_pe_permute(ctx, dst);
++            break;
++        case GGML_OP_MODULATE_ROWS:
++            ggml_cuda_op_modulate_rows(ctx, dst);
++            break;
++        case GGML_OP_SWIGLU_SCALED:
++            ggml_cuda_op_swiglu_scaled(ctx, dst);
++            break;
+         case GGML_OP_CROSS_ENTROPY_LOSS:
+             ggml_cuda_cross_entropy_loss(ctx, dst);
+             break;
+@@ -6784,6 +6795,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
+             return ggml_cuda_sage_attn_supported(dev_ctx->device, op);
+         case GGML_OP_SOL_ATTN:
+             return ggml_cuda_sol_attn_supported(dev_ctx->device, op);
++        case GGML_OP_ROPE_PE_PERMUTE:
++            return ggml_cuda_rope_pe_permute_supported(op);
++        case GGML_OP_MODULATE_ROWS:
++            return ggml_cuda_modulate_rows_supported(op);
++        case GGML_OP_SWIGLU_SCALED:
++            return ggml_cuda_swiglu_scaled_supported(op);
+         case GGML_OP_CROSS_ENTROPY_LOSS:
+         case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
+         case GGML_OP_OPT_STEP_ADAMW:
+diff --git a/src/ggml-cuda/modulate-rows.cu b/src/ggml-cuda/modulate-rows.cu
+new file mode 100644
+index 00000000..1b639d28
+--- /dev/null
++++ b/src/ggml-cuda/modulate-rows.cu
+@@ -0,0 +1,168 @@
++#include "modulate-rows.cuh"
++#include "unary.cuh"
++
++// Rounding matches the unfused graph: __fmul_rn/__fadd_rn are the separately rounded MUL and ADD
++// kernels (never contracted into an FMA), `s*y + bias` is the expression the SCALE kernel
++// evaluates, and the swiglu product is the GLU kernel's silu(a)*b.
++
++template <int mode, bool has_scale>
++static __global__ void modulate_rows_kernel(
++        const char * __restrict__ a, const char * __restrict__ b, const char * __restrict__ c, char * __restrict__ dst,
++        const int nc, const int64_t nr1, const int64_t ne2, const int64_t row0,
++        const int64_t nba1, const int64_t nba2, const int64_t nba3,
++        const int64_t nbb1, const int64_t nbb2, const int64_t nbb3,
++        const int64_t nbd1, const int64_t nbd2, const int64_t nbd3,
++        const float s, const float bias) {
++    const int64_t ir = blockIdx.x;
++    const int64_t i1 = row0 + ir % nr1;
++    const int64_t i2 = (ir / nr1) % ne2;
++    const int64_t i3 = ir / (nr1 * ne2);
++
++    const float * xa = (const float *) (a + i1 * nba1 + i2 * nba2 + i3 * nba3);
++    float       * y  = (float *) (dst + i1 * nbd1 + i2 * nbd2 + i3 * nbd3);
++
++    if constexpr (mode == 0) {
++        const float * scale = (const float *) b;
++        const float * shift = (const float *) c;
++        for (int i = threadIdx.x; i < nc; i += blockDim.x) {
++            const float x = xa[i];
++            float v = __fadd_rn(__fadd_rn(x, __fmul_rn(x, scale[i])), shift[i]);
++            if constexpr (has_scale) {
++                v = s * v + bias;
++            }
++            y[i] = v;
++        }
++    } else {
++        const float * u    = (const float *) (b + i1 * nbb1 + i2 * nbb2 + i3 * nbb3);
++        const float * gate = (const float *) c;
++        for (int i = threadIdx.x; i < nc; i += blockDim.x) {
++            float w = u[i];
++            if constexpr (has_scale) {
++                w = s * w + bias;
++            }
++            y[i] = __fadd_rn(xa[i], __fmul_rn(w, gate[i]));
++        }
++    }
++}
++
++void ggml_cuda_op_modulate_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
++    const ggml_tensor * a = dst->src[0];
++    const ggml_tensor * b = dst->src[1];
++    const ggml_tensor * c = dst->src[2];
++
++    const int     mode = ggml_get_op_params_i32(dst, 0);
++    const int64_t row0 = ggml_get_op_params_i32(dst, 1);
++    const int64_t row1 = ggml_get_op_params_i32(dst, 2);
++    const float   s    = ggml_get_op_params_f32(dst, 3);
++
++    const int64_t nr1   = row1 - row0;
++    const int64_t nrows = nr1 * a->ne[2] * a->ne[3];
++    const int     nc    = (int) a->ne[0];
++    if (nrows == 0) {
++        return;
++    }
++
++    cudaStream_t stream = ctx.stream();
++    const dim3   grid((unsigned int) nrows, 1, 1);
++    const dim3   block(256, 1, 1);
++
++#define MODULATE_ROWS_LAUNCH(MODE, HAS_SCALE)                                                                   \
++    modulate_rows_kernel<MODE, HAS_SCALE><<<grid, block, 0, stream>>>(                                          \
++        (const char *) a->data, (const char *) b->data, (const char *) c->data, (char *) dst->data, nc, nr1,    \
++        a->ne[2], row0, a->nb[1], a->nb[2], a->nb[3], b->nb[1], b->nb[2], b->nb[3], dst->nb[1], dst->nb[2],     \
++        dst->nb[3], s, 0.0f)
++
++    if (mode == 0) {
++        if (s != 1.0f) {
++            MODULATE_ROWS_LAUNCH(0, true);
++        } else {
++            MODULATE_ROWS_LAUNCH(0, false);
++        }
++    } else {
++        if (s != 1.0f) {
++            MODULATE_ROWS_LAUNCH(1, true);
++        } else {
++            MODULATE_ROWS_LAUNCH(1, false);
++        }
++    }
++#undef MODULATE_ROWS_LAUNCH
++}
++
++bool ggml_cuda_modulate_rows_supported(const ggml_tensor * op) {
++    const int64_t nrows = (int64_t) (ggml_get_op_params_i32(op, 2) - ggml_get_op_params_i32(op, 1)) * op->src[0]->ne[2] * op->src[0]->ne[3];
++    return op->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op) && op->src[0]->ne[0] <= INT_MAX && nrows <= INT_MAX &&
++           ggml_is_contiguous(op->src[ggml_get_op_params_i32(op, 0) == 0 ? 1 : 2]) &&
++           (ggml_get_op_params_i32(op, 0) != 0 || ggml_is_contiguous(op->src[2]));
++}
++
++template <bool has_in, bool has_out>
++static __global__ void swiglu_scaled_kernel(
++        const char * __restrict__ a, const char * __restrict__ b, float * __restrict__ dst,
++        const int nc, const int64_t ne1, const int64_t ne2,
++        const int64_t nba1, const int64_t nba2, const int64_t nba3,
++        const int64_t nbb1, const int64_t nbb2, const int64_t nbb3,
++        const float in_scale, const float out_scale, const float bias) {
++    const int64_t ir = blockIdx.x;
++    const int64_t i1 = ir % ne1;
++    const int64_t i2 = (ir / ne1) % ne2;
++    const int64_t i3 = ir / (ne1 * ne2);
++
++    const float * xa = (const float *) (a + i1 * nba1 + i2 * nba2 + i3 * nba3);
++    const float * xb = (const float *) (b + i1 * nbb1 + i2 * nbb2 + i3 * nbb3);
++    float       * y  = dst + ir * nc;
++
++    for (int i = threadIdx.x; i < nc; i += blockDim.x) {
++        float va = xa[i];
++        float vb = xb[i];
++        if constexpr (has_in) {
++            va = in_scale * va + bias;
++            vb = in_scale * vb + bias;
++        }
++        float v = ggml_cuda_op_silu_single(va) * vb;
++        if constexpr (has_out) {
++            v = out_scale * v + bias;
++        }
++        y[i] = v;
++    }
++}
++
++void ggml_cuda_op_swiglu_scaled(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
++    const ggml_tensor * a = dst->src[0];
++    const ggml_tensor * b = dst->src[1];
++
++    const float in_scale  = ggml_get_op_params_f32(dst, 0);
++    const float out_scale = ggml_get_op_params_f32(dst, 1);
++
++    GGML_ASSERT(ggml_is_contiguous(dst));
++    const int64_t nrows = ggml_nrows(a);
++    const int     nc    = (int) a->ne[0];
++    if (nrows == 0) {
++        return;
++    }
++
++    cudaStream_t stream = ctx.stream();
++    const dim3   grid((unsigned int) nrows, 1, 1);
++    const dim3   block(256, 1, 1);
++
++#define SWIGLU_SCALED_LAUNCH(HAS_IN, HAS_OUT)                                                                  \
++    swiglu_scaled_kernel<HAS_IN, HAS_OUT><<<grid, block, 0, stream>>>(                                         \
++        (const char *) a->data, (const char *) b->data, (float *) dst->data, nc, a->ne[1], a->ne[2], a->nb[1], \
++        a->nb[2], a->nb[3], b->nb[1], b->nb[2], b->nb[3], in_scale, out_scale, 0.0f)
++
++    const bool has_in  = in_scale != 1.0f;
++    const bool has_out = out_scale != 1.0f;
++    if (has_in && has_out) {
++        SWIGLU_SCALED_LAUNCH(true, true);
++    } else if (has_in) {
++        SWIGLU_SCALED_LAUNCH(true, false);
++    } else if (has_out) {
++        SWIGLU_SCALED_LAUNCH(false, true);
++    } else {
++        SWIGLU_SCALED_LAUNCH(false, false);
++    }
++#undef SWIGLU_SCALED_LAUNCH
++}
++
++bool ggml_cuda_swiglu_scaled_supported(const ggml_tensor * op) {
++    return op->type == GGML_TYPE_F32 && op->src[0]->ne[0] <= INT_MAX && ggml_nrows(op->src[0]) <= INT_MAX;
++}
+diff --git a/src/ggml-cuda/modulate-rows.cuh b/src/ggml-cuda/modulate-rows.cuh
+new file mode 100644
+index 00000000..216d0767
+--- /dev/null
++++ b/src/ggml-cuda/modulate-rows.cuh
+@@ -0,0 +1,7 @@
++#include "common.cuh"
++
++void ggml_cuda_op_modulate_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
++bool ggml_cuda_modulate_rows_supported(const ggml_tensor * op);
++
++void ggml_cuda_op_swiglu_scaled(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
++bool ggml_cuda_swiglu_scaled_supported(const ggml_tensor * op);
+diff --git a/src/ggml-cuda/rope-pe.cu b/src/ggml-cuda/rope-pe.cu
+new file mode 100644
+index 00000000..3afe4423
+--- /dev/null
++++ b/src/ggml-cuda/rope-pe.cu
+@@ -0,0 +1,102 @@
++#include "rope-pe.cuh"
++#include "convert.cuh"
++
++// Fused rotary embedding (precomputed cos/sin table) + head-major relayout + scale + cast.
++// One block per (token, batch); the block walks every head so the table entries are read once.
++// Rounding matches the unfused graph exactly: __fmul_rn/__fadd_rn keep the two products and the
++// sum as separately rounded operations (no FMA contraction), the scale is the same `scale*y + bias`
++// expression the SCALE kernel evaluates, and the conversion uses the same ggml_cuda_cast as CPY.
++template <typename dst_t, bool has_scale>
++static __global__ void rope_pe_permute_kernel(
++        const char * __restrict__ x, const float * __restrict__ pe, dst_t * __restrict__ dst,
++        const int D, const int H, const int64_t L, const int n_rot,
++        const int64_t nb01, const int64_t nb02, const int64_t nb03,
++        const float scale, const float bias) {
++    const int64_t l    = blockIdx.x;
++    const int64_t n    = blockIdx.y;
++    const int     half = n_rot / 2;
++
++    for (int i = threadIdx.x; i < D; i += blockDim.x) {
++        const bool rot = i < n_rot;
++        int   j  = i;
++        float c0 = 0.0f;
++        float c1 = 0.0f;
++        if (rot) {
++            const int r = i >= half;
++            j           = i - r * half;
++            const float * t = pe + (l * half + j) * 4 + 2 * r;
++            c0 = t[0];
++            c1 = t[1];
++        }
++        for (int h = 0; h < H; ++h) {
++            const float * xr = (const float *) (x + h * nb01 + l * nb02 + n * nb03);
++            float y;
++            if (rot) {
++                y = __fadd_rn(__fmul_rn(xr[j], c0), __fmul_rn(xr[j + half], c1));
++            } else {
++                y = xr[i];
++            }
++            if constexpr (has_scale) {
++                y = scale * y + bias;
++            }
++            dst[((n * H + h) * L + l) * D + i] = ggml_cuda_cast<dst_t>(y);
++        }
++    }
++}
++
++template <typename dst_t>
++static void rope_pe_permute_cuda(const ggml_tensor * src0, const float * pe, dst_t * dst, const int n_rot,
++                                 const float scale, cudaStream_t stream) {
++    const int     D = (int) src0->ne[0];
++    const int     H = (int) src0->ne[1];
++    const int64_t L = src0->ne[2];
++    const int64_t N = src0->ne[3];
++
++    const dim3 grid((unsigned int) L, (unsigned int) N, 1);
++    const dim3 block(D >= 128 ? 128 : ((D + 31) / 32) * 32, 1, 1);
++    const char * x = (const char *) src0->data;
++    if (scale != 1.0f) {
++        rope_pe_permute_kernel<dst_t, true><<<grid, block, 0, stream>>>(
++            x, pe, dst, D, H, L, n_rot, src0->nb[1], src0->nb[2], src0->nb[3], scale, 0.0f);
++    } else {
++        rope_pe_permute_kernel<dst_t, false><<<grid, block, 0, stream>>>(
++            x, pe, dst, D, H, L, n_rot, src0->nb[1], src0->nb[2], src0->nb[3], scale, 0.0f);
++    }
++}
++
++void ggml_cuda_op_rope_pe_permute(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
++    const ggml_tensor * src0 = dst->src[0];
++    const ggml_tensor * pe   = dst->src[1];
++
++    const int   n_rot = ggml_get_op_params_i32(dst, 0);
++    const float scale = ggml_get_op_params_f32(dst, 1);
++
++    GGML_ASSERT(src0->type == GGML_TYPE_F32 && src0->nb[0] == sizeof(float));
++    GGML_ASSERT(ggml_is_contiguous(dst));
++    GGML_ASSERT(n_rot == 0 || (pe != nullptr && ggml_is_contiguous(pe)));
++
++    cudaStream_t  stream = ctx.stream();
++    const float * pe_d   = pe ? (const float *) pe->data : nullptr;
++
++    if (dst->type == GGML_TYPE_F16) {
++        rope_pe_permute_cuda(src0, pe_d, (half *) dst->data, n_rot, scale, stream);
++    } else {
++        GGML_ASSERT(dst->type == GGML_TYPE_F32);
++        rope_pe_permute_cuda(src0, pe_d, (float *) dst->data, n_rot, scale, stream);
++    }
++}
++
++bool ggml_cuda_rope_pe_permute_supported(const ggml_tensor * op) {
++    const ggml_tensor * src0 = op->src[0];
++    const ggml_tensor * pe   = op->src[1];
++    if (src0->type != GGML_TYPE_F32 || src0->nb[0] != sizeof(float)) {
++        return false;
++    }
++    if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) {
++        return false;
++    }
++    if (src0->ne[2] > INT_MAX || src0->ne[3] > 65535 || src0->ne[1] > INT_MAX || src0->ne[0] > INT_MAX) {
++        return false;
++    }
++    return pe == nullptr || (pe->type == GGML_TYPE_F32 && ggml_is_contiguous(pe));
++}
+diff --git a/src/ggml-cuda/rope-pe.cuh b/src/ggml-cuda/rope-pe.cuh
+new file mode 100644
+index 00000000..b68b6af4
+--- /dev/null
++++ b/src/ggml-cuda/rope-pe.cuh
+@@ -0,0 +1,5 @@
++#include "common.cuh"
++
++void ggml_cuda_op_rope_pe_permute(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
++
++bool ggml_cuda_rope_pe_permute_supported(const ggml_tensor * op);
+diff --git a/src/ggml.c b/src/ggml.c
+index da9ccada..a9f0b47d 100644
+--- a/src/ggml.c
++++ b/src/ggml.c
+@@ -1129,9 +1129,12 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
+     "SAGE_ATTN",
+     "SOL_ATTN",
+     "ROPE_APPLY",
++    "ROPE_PE_PERMUTE",
++    "MODULATE_ROWS",
++    "SWIGLU_SCALED",
+ };
+ 
+-static_assert(GGML_OP_COUNT == 105, "GGML_OP_COUNT != 105");
++static_assert(GGML_OP_COUNT == 108, "GGML_OP_COUNT != 108");
+ 
+ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
+     "none",
+@@ -1249,9 +1252,12 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
+     "sage_attn(x,y,z)",
+     "sol_attn(x,y,z)",
+     "rope_apply(x,pe)",
++    "rope_pe_permute(x,pe)",
++    "modulate_rows(x,a,b)",
++    "swiglu_scaled(x,y)",
+ };
+ 
+-static_assert(GGML_OP_COUNT == 105, "GGML_OP_COUNT != 105");
++static_assert(GGML_OP_COUNT == 108, "GGML_OP_COUNT != 108");
+ 
+ static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
+ 
+@@ -5721,6 +5727,125 @@ struct ggml_tensor* ggml_sol_attn(
+     return result;
+ }
+ 
++struct ggml_tensor * ggml_rope_pe_permute(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * a,
++        struct ggml_tensor  * pe,
++        int                   n_rot,
++        float                 scale,
++        enum ggml_type        type) {
++    GGML_ASSERT(a->type == GGML_TYPE_F32);
++    GGML_ASSERT(a->nb[0] == ggml_type_size(a->type));
++    GGML_ASSERT(type == GGML_TYPE_F32 || type == GGML_TYPE_F16);
++    GGML_ASSERT(n_rot >= 0 && n_rot % 2 == 0 && n_rot <= a->ne[0]);
++    if (n_rot > 0) {
++        GGML_ASSERT(pe != NULL && pe->type == GGML_TYPE_F32 && ggml_is_contiguous(pe));
++        GGML_ASSERT(pe->ne[0] == 2 && pe->ne[1] == 2 && pe->ne[2] == n_rot / 2 && pe->ne[3] == a->ne[2]);
++    } else {
++        GGML_ASSERT(pe == NULL);
++    }
++    GGML_ASSERT(isfinite(scale));
++
++    struct ggml_tensor * result = ggml_new_tensor_4d(ctx, type, a->ne[0], a->ne[2], a->ne[1], a->ne[3]);
++
++    ggml_set_op_params_i32(result, 0, n_rot);
++    ggml_set_op_params_f32(result, 1, scale);
++
++    result->op     = GGML_OP_ROPE_PE_PERMUTE;
++    result->src[0] = a;
++    result->src[1] = pe;
++
++    return result;
++}
++
++static struct ggml_tensor * ggml_modulate_rows_impl(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * acc,
++        struct ggml_tensor  * a,
++        struct ggml_tensor  * b,
++        struct ggml_tensor  * c,
++        int64_t               row0,
++        int64_t               row1,
++        float                 s,
++        int32_t               mode) {
++    GGML_ASSERT(a->type == GGML_TYPE_F32 && a->nb[0] == sizeof(float));
++    GGML_ASSERT(row0 >= 0 && row0 < row1 && row1 <= a->ne[1] && row1 <= INT32_MAX);
++    GGML_ASSERT(isfinite(s));
++    // mode 0: a = x, b = scale, c = shift; mode 1: a = base, b = u, c = gate
++    struct ggml_tensor * vec0 = mode == 0 ? b : c;
++    GGML_ASSERT(vec0->type == GGML_TYPE_F32 && vec0->nb[0] == sizeof(float) && ggml_nelements(vec0) == a->ne[0]);
++    if (mode == 0) {
++        GGML_ASSERT(c->type == GGML_TYPE_F32 && c->nb[0] == sizeof(float) && ggml_nelements(c) == a->ne[0]);
++    } else {
++        GGML_ASSERT(b->type == GGML_TYPE_F32 && b->nb[0] == sizeof(float) && ggml_are_same_shape(a, b));
++    }
++    if (acc != NULL) {
++        GGML_ASSERT(acc->type == GGML_TYPE_F32 && ggml_is_contiguous(acc) && ggml_are_same_shape(acc, a));
++    }
++
++    struct ggml_tensor * result = acc != NULL ? ggml_view_tensor(ctx, acc) : ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, a->ne);
++
++    ggml_set_op_params_i32(result, 0, mode);
++    ggml_set_op_params_i32(result, 1, (int32_t) row0);
++    ggml_set_op_params_i32(result, 2, (int32_t) row1);
++    ggml_set_op_params_f32(result, 3, s);
++
++    result->op     = GGML_OP_MODULATE_ROWS;
++    result->src[0] = a;
++    result->src[1] = b;
++    result->src[2] = c;
++    result->src[3] = acc;
++
++    return result;
++}
++
++struct ggml_tensor * ggml_modulate_rows(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * acc,
++        struct ggml_tensor  * x,
++        struct ggml_tensor  * scale,
++        struct ggml_tensor  * shift,
++        int64_t               row0,
++        int64_t               row1,
++        float                 out_scale) {
++    return ggml_modulate_rows_impl(ctx, acc, x, scale, shift, row0, row1, out_scale, 0);
++}
++
++struct ggml_tensor * ggml_gated_add_rows(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * acc,
++        struct ggml_tensor  * base,
++        struct ggml_tensor  * u,
++        struct ggml_tensor  * gate,
++        int64_t               row0,
++        int64_t               row1,
++        float                 u_scale) {
++    return ggml_modulate_rows_impl(ctx, acc, base, u, gate, row0, row1, u_scale, 1);
++}
++
++struct ggml_tensor * ggml_swiglu_scaled(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * a,
++        struct ggml_tensor  * b,
++        float                 in_scale,
++        float                 out_scale) {
++    GGML_ASSERT(a->type == GGML_TYPE_F32 && b->type == GGML_TYPE_F32);
++    GGML_ASSERT(a->nb[0] == sizeof(float) && b->nb[0] == sizeof(float));
++    GGML_ASSERT(ggml_are_same_shape(a, b));
++    GGML_ASSERT(isfinite(in_scale) && isfinite(out_scale));
++
++    struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, GGML_MAX_DIMS, a->ne);
++
++    ggml_set_op_params_f32(result, 0, in_scale);
++    ggml_set_op_params_f32(result, 1, out_scale);
++
++    result->op     = GGML_OP_SWIGLU_SCALED;
++    result->src[0] = a;
++    result->src[1] = b;
++
++    return result;
++}
++
+ void ggml_flash_attn_ext_set_prec(
+         struct ggml_tensor * a,
+         enum ggml_prec       prec) {
+diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
+index 83136355..fdda5dc7 100644
+--- a/tests/test-backend-ops.cpp
++++ b/tests/test-backend-ops.cpp
+@@ -2192,6 +2192,118 @@ struct test_glu : public test_case {
+     }
+ };
+ 
++// GGML_OP_ROPE_PE_PERMUTE
++struct test_rope_pe_permute : public test_case {
++    const ggml_type type;
++    const std::array<int64_t, 4> ne_a; // [head_dim, n_head, n_tokens, n_batch]
++    const int n_rot;
++    const float scale;
++    const bool view; // a is a strided view (one third of a fused qkv projection)
++
++    std::string vars() override {
++        return VARS_TO_STR5(type, ne_a, n_rot, scale, view);
++    }
++
++    test_rope_pe_permute(ggml_type type = GGML_TYPE_F32,
++            std::array<int64_t, 4> ne_a = {128, 4, 9, 1},
++            int n_rot = 96, float scale = 1.0f, bool view = false)
++        : type(type), ne_a(ne_a), n_rot(n_rot), scale(scale), view(view) {}
++
++    ggml_tensor * build_graph(ggml_context * ctx) override {
++        ggml_tensor * a;
++        if (view) {
++            ggml_tensor * qkv = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 3 * ne_a[0] * ne_a[1], ne_a[2], ne_a[3], 1);
++            ggml_set_name(qkv, "qkv");
++            a = ggml_view_4d(ctx, qkv, ne_a[0], ne_a[1], ne_a[2], ne_a[3],
++                             ne_a[0] * sizeof(float), qkv->nb[1], qkv->nb[2], ne_a[0] * ne_a[1] * sizeof(float));
++            ggml_set_name(a, "view_of_qkv");
++        } else {
++            a = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne_a.data());
++            ggml_set_name(a, "a");
++        }
++        ggml_tensor * pe = nullptr;
++        if (n_rot > 0) {
++            pe = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 2, 2, n_rot / 2, ne_a[2]);
++            ggml_set_name(pe, "pe");
++        }
++        ggml_tensor * out = ggml_rope_pe_permute(ctx, a, pe, n_rot, scale, type);
++        ggml_set_name(out, "out");
++        return out;
++    }
++};
++
++// GGML_OP_MODULATE_ROWS (a chain of segments written in place)
++struct test_modulate_rows : public test_case {
++    const int mode; // 0: modulate, 1: gated add
++    const std::array<int64_t, 4> ne;
++    const float s;
++    const bool view;
++
++    std::string vars() override {
++        return VARS_TO_STR4(mode, ne, s, view);
++    }
++
++    test_modulate_rows(int mode = 0, std::array<int64_t, 4> ne = {64, 12, 2, 1}, float s = 1.0f, bool view = false)
++        : mode(mode), ne(ne), s(s), view(view) {}
++
++    ggml_tensor * build_graph(ggml_context * ctx) override {
++        auto make = [&](const char * name) {
++            ggml_tensor * t;
++            if (view) {
++                ggml_tensor * big = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 2 * ne[0], ne[1], ne[2], ne[3]);
++                t = ggml_view_4d(ctx, big, ne[0], ne[1], ne[2], ne[3], big->nb[1], big->nb[2], big->nb[3], ne[0] * sizeof(float));
++            } else {
++                t = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne.data());
++            }
++            ggml_set_name(t, name);
++            return t;
++        };
++        ggml_tensor * a = make("a");
++        ggml_tensor * u = mode == 1 ? make("u") : nullptr;
++        // modulation table: 3 rows of 2 vectors each, taken as views like an adaLN projection
++        ggml_tensor * table = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2 * ne[0], 3);
++        ggml_set_name(table, "table");
++        const int64_t bounds[4] = {0, ne[1] / 4, ne[1] - 2, ne[1]};
++        ggml_tensor * out = nullptr;
++        for (int seg = 0; seg < 3; ++seg) {
++            const int row = (seg + 1) % 3;
++            ggml_tensor * v0 = ggml_view_1d(ctx, table, ne[0], row * table->nb[1]);
++            ggml_tensor * v1 = ggml_view_1d(ctx, table, ne[0], row * table->nb[1] + ne[0] * sizeof(float));
++            if (mode == 0) {
++                out = ggml_modulate_rows(ctx, out, a, v0, v1, bounds[seg], bounds[seg + 1], s);
++            } else {
++                out = ggml_gated_add_rows(ctx, out, a, u, v0, bounds[seg], bounds[seg + 1], s);
++            }
++        }
++        ggml_set_name(out, "out");
++        return out;
++    }
++};
++
++// GGML_OP_SWIGLU_SCALED
++struct test_swiglu_scaled : public test_case {
++    const std::array<int64_t, 4> ne;
++    const float in_scale;
++    const float out_scale;
++
++    std::string vars() override {
++        return VARS_TO_STR3(ne, in_scale, out_scale);
++    }
++
++    test_swiglu_scaled(std::array<int64_t, 4> ne = {64, 5, 3, 1}, float in_scale = 1.0f, float out_scale = 1.0f)
++        : ne(ne), in_scale(in_scale), out_scale(out_scale) {}
++
++    ggml_tensor * build_graph(ggml_context * ctx) override {
++        ggml_tensor * h = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 2 * ne[0], ne[1], ne[2], ne[3]);
++        ggml_set_name(h, "h");
++        ggml_tensor * a = ggml_view_4d(ctx, h, ne[0], ne[1], ne[2], ne[3], h->nb[1], h->nb[2], h->nb[3], 0);
++        ggml_tensor * b = ggml_view_4d(ctx, h, ne[0], ne[1], ne[2], ne[3], h->nb[1], h->nb[2], h->nb[3], ne[0] * sizeof(float));
++        ggml_tensor * out = ggml_swiglu_scaled(ctx, a, b, in_scale, out_scale);
++        ggml_set_name(out, "out");
++        return out;
++    }
++};
++
+ struct test_glu_split : public test_case {
+     const ggml_glu_op op;
+     const ggml_type type;
+@@ -9759,6 +9871,33 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
+         }
+     }
+ 
++    for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
++        for (bool view : {false, true}) {
++            for (float scale : {1.0f, 1.0f / 128.0f}) {
++                test_cases.emplace_back(new test_rope_pe_permute(type, {128, 4, 9, 1}, 96, scale, view));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {128, 3, 5, 2}, 128, scale, view));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {128, 4, 9, 1}, 0, scale, view));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {64, 3, 7, 2}, 32, scale, view));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {200, 2, 3, 1}, 100, scale, view));
++            }
++        }
++    }
++
++    for (int mode : {0, 1}) {
++        for (bool view : {false, true}) {
++            for (float sc : {1.0f, mode == 0 ? 1.0f / 128.0f : 128.0f}) {
++                test_cases.emplace_back(new test_modulate_rows(mode, {64, 12, 2, 1}, sc, view));
++                test_cases.emplace_back(new test_modulate_rows(mode, {300, 9, 1, 2}, sc, view));
++            }
++        }
++    }
++    for (float in_scale : {1.0f, 128.0f}) {
++        for (float out_scale : {1.0f, 1.0f / 128.0f}) {
++            test_cases.emplace_back(new test_swiglu_scaled({64, 5, 3, 1}, in_scale, out_scale));
++            test_cases.emplace_back(new test_swiglu_scaled({300, 7, 1, 2}, in_scale, out_scale));
++        }
++    }
++
+     for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
+         for (int v : {0, 1}) {
+             for (float alpha : {.5f, 1.702f}) {
diff --git a/scripts/unsloth/ggml-patches/0005-ggml-rope-pe-permute-rms.patch b/scripts/unsloth/ggml-patches/0005-ggml-rope-pe-permute-rms.patch
new file mode 100644
index 0000000000..e5f2231ead
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0005-ggml-rope-pe-permute-rms.patch
@@ -0,0 +1,304 @@
+From b8666f9e06e9e07255b8bfc92c42988716b7b865 Mon Sep 17 00:00:00 2001
+From: Daniel Han <danielhanchen@gmail.com>
+Date: Thu, 1 Oct 2026 23:16:25 -0700
+Subject: [PATCH] ggml : rope_pe_permute with the per-head RMS norm folded in
+
+ggml_rope_pe_permute_rms(a, pe, n_rot, scale, type, eps) is ggml_rope_pe_permute of
+ggml_rms_norm(a, eps) without a weight. CPU uses rms_norm's arithmetic (double sum,
+1/sqrtf, vec scale); CUDA normalises each head row with the block size and block
+reduction rms_norm_f32_cuda's small-row path uses for that row length, so both match
+the two-op chain bit for bit. CUDA supports it for head_dim <= 128 when that small-row
+path is active (not HIP/MUSA, GGML_CUDA_NORM_SMALL_ROWS not 0); elsewhere the op is
+reported unsupported and callers keep the separate norm. test-backend-ops cases added.
+---
+ include/ggml.h             |  12 ++++
+ src/ggml-cpu/ops.cpp       |  17 +++++-
+ src/ggml-cuda/rope-pe.cu   | 120 +++++++++++++++++++++++++++++++++++++
+ src/ggml.c                 |  15 +++++
+ tests/test-backend-ops.cpp |  15 +++--
+ 5 files changed, 174 insertions(+), 5 deletions(-)
+
+[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is
+ the whole patch as one diff, regenerated from the port.]
+---
+diff --git a/include/ggml.h b/include/ggml.h
+index bbafe175..f7ef1573 100644
+--- a/include/ggml.h
++++ b/include/ggml.h
+@@ -2578,6 +2578,18 @@ extern "C" {
+             float                 scale,
+             enum ggml_type        type);
+ 
++    // ggml_rope_pe_permute of ggml_rms_norm(a, eps) (no weight): every head row of `a` is
++    // RMS-normalised first, with the same arithmetic as GGML_OP_RMS_NORM on the same backend, so
++    // the result matches the two-op chain bit for bit where the backend supports it.
++    GGML_API struct ggml_tensor * ggml_rope_pe_permute_rms(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * a,
++            struct ggml_tensor  * pe,
++            int                   n_rot,
++            float                 scale,
++            enum ggml_type        type,
++            float                 eps);
++
+     // Row-range modulation for adaLN-style blocks whose sequence is split into segments that use
+     // different modulation vectors. Writes rows [row0, row1) along dim 1 (all of dims 2 and 3) of the
+     // result and leaves every other row as it is in `acc`: chain one call per segment, passing the
+diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp
+index 94333d30..ebe6e202 100644
+--- a/src/ggml-cpu/ops.cpp
++++ b/src/ggml-cpu/ops.cpp
+@@ -12701,6 +12701,8 @@ void ggml_compute_forward_rope_pe_permute(
+ 
+     const int   n_rot = ggml_get_op_params_i32(dst, 0);
+     const float scale = ggml_get_op_params_f32(dst, 1);
++    const bool  norm  = ggml_get_op_params_i32(dst, 2) != 0;
++    const float eps   = ggml_get_op_params_f32(dst, 3);
+     const int   half  = n_rot / 2;
+ 
+     const int64_t D = src0->ne[0];
+@@ -12715,7 +12717,7 @@ void ggml_compute_forward_rope_pe_permute(
+ 
+     // Same rounding points as the unfused graph: each product, the sum, the scale and the
+     // conversion are separate passes, so no contraction into fused multiply-adds can happen.
+-    std::vector<float> p0(n_rot), p1(n_rot), y(D);
++    std::vector<float> p0(n_rot), p1(n_rot), y(D), xn(norm ? D : 0);
+ 
+     for (int64_t ir = ir0; ir < ir1; ++ir) {
+         const int64_t h = ir % H;
+@@ -12724,6 +12726,19 @@ void ggml_compute_forward_rope_pe_permute(
+ 
+         const float * x = (const float *) ((const char *) src0->data + h * src0->nb[1] + l * src0->nb[2] + n * src0->nb[3]);
+ 
++        if (norm) {
++            // ggml_compute_forward_rms_norm_f32 without a weight
++            ggml_float sum = 0.0;
++            for (int64_t i = 0; i < D; i++) {
++                sum += (ggml_float)(x[i] * x[i]);
++            }
++            const float mean = sum/D;
++            const float s    = 1.0f/sqrtf(mean + eps);
++            memcpy(xn.data(), x, D * sizeof(float));
++            ggml_vec_scale_f32((int) D, xn.data(), s);
++            x = xn.data();
++        }
++
+         if (n_rot > 0) {
+             const float * t = (const float *) pe->data + l * (int64_t) half * 4;
+             for (int i = 0; i < n_rot; ++i) {
+diff --git a/src/ggml-cuda/rope-pe.cu b/src/ggml-cuda/rope-pe.cu
+index 3afe4423..b6dd7a69 100644
+--- a/src/ggml-cuda/rope-pe.cu
++++ b/src/ggml-cuda/rope-pe.cu
+@@ -44,6 +44,112 @@ static __global__ void rope_pe_permute_kernel(
+     }
+ }
+ 
++// Same op with every head row RMS-normalised first (no weight). One block of block_size threads
++// per (token, batch), the size and reduction rms_norm_f32_cuda's small-row path uses for this row
++// length, so each normalised value is bit-identical to GGML_OP_RMS_NORM's; the normalised row is
++// staged in shared memory for the RoPE partner reads.
++template <typename dst_t, bool has_scale, int block_size>
++static __global__ void rope_pe_permute_rms_kernel(
++        const char * __restrict__ x, const float * __restrict__ pe, dst_t * __restrict__ dst,
++        const int D, const int H, const int64_t L, const int n_rot,
++        const int64_t nb01, const int64_t nb02, const int64_t nb03,
++        const float scale, const float bias, const float eps) {
++    const int64_t l    = blockIdx.x;
++    const int64_t n    = blockIdx.y;
++    const int     half = n_rot / 2;
++    const int     i    = threadIdx.x;
++
++    __shared__ float s_sum[32];
++    __shared__ float xs[block_size];
++
++    const bool rot = i < n_rot;
++    int   j  = i;
++    float c0 = 0.0f;
++    float c1 = 0.0f;
++    if (rot) {
++        const int r = i >= half;
++        j           = i - r * half;
++        const float * t = pe + (l * half + j) * 4 + 2 * r;
++        c0 = t[0];
++        c1 = t[1];
++    }
++    for (int h = 0; h < H; ++h) {
++        const float * xr = (const float *) (x + h * nb01 + l * nb02 + n * nb03);
++        float xi  = 0.0f;
++        float tmp = 0.0f;
++        if (i < D) {
++            xi = xr[i];
++            tmp += xi * xi;
++        }
++        tmp = block_reduce<block_reduce_method::SUM, block_size>(tmp, s_sum);
++        const float mean = tmp / D;
++        const float sc   = rsqrtf(mean + eps);
++        if (i < D) {
++            xs[i] = sc * xi;
++        }
++        __syncthreads();
++        if (i < D) {
++            float y;
++            if (rot) {
++                y = __fadd_rn(__fmul_rn(xs[j], c0), __fmul_rn(xs[j + half], c1));
++            } else {
++                y = xs[i];
++            }
++            if constexpr (has_scale) {
++                y = scale * y + bias;
++            }
++            dst[((n * H + h) * L + l) * D + i] = ggml_cuda_cast<dst_t>(y);
++        }
++        __syncthreads();
++    }
++}
++
++// rms_norm_f32_cuda takes its small-row path (and so its reduction order) only here.
++static bool rope_pe_rms_small_rows_enabled() {
++#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
++    return false;
++#else
++    static const bool enabled = [] {
++        const char * e = getenv("GGML_CUDA_NORM_SMALL_ROWS");
++        return e == nullptr || atoi(e) != 0;
++    }();
++    return enabled && WARP_SIZE == 32;
++#endif
++}
++
++template <typename dst_t, int block_size>
++static void rope_pe_permute_rms_launch(const ggml_tensor * src0, const float * pe, dst_t * dst, const int n_rot,
++                                       const float scale, const float eps, cudaStream_t stream) {
++    const int     D = (int) src0->ne[0];
++    const int     H = (int) src0->ne[1];
++    const int64_t L = src0->ne[2];
++    const int64_t N = src0->ne[3];
++    const dim3 grid((unsigned int) L, (unsigned int) N, 1);
++    const dim3 block(block_size, 1, 1);
++    const char * x = (const char *) src0->data;
++    if (scale != 1.0f) {
++        rope_pe_permute_rms_kernel<dst_t, true, block_size><<<grid, block, 0, stream>>>(
++            x, pe, dst, D, H, L, n_rot, src0->nb[1], src0->nb[2], src0->nb[3], scale, 0.0f, eps);
++    } else {
++        rope_pe_permute_rms_kernel<dst_t, false, block_size><<<grid, block, 0, stream>>>(
++            x, pe, dst, D, H, L, n_rot, src0->nb[1], src0->nb[2], src0->nb[3], scale, 0.0f, eps);
++    }
++}
++
++template <typename dst_t>
++static void rope_pe_permute_rms_cuda(const ggml_tensor * src0, const float * pe, dst_t * dst, const int n_rot,
++                                     const float scale, const float eps, cudaStream_t stream) {
++    const int D = (int) src0->ne[0];
++    GGML_ASSERT(D <= 128);
++    if (D <= 32) {
++        rope_pe_permute_rms_launch<dst_t, 32>(src0, pe, dst, n_rot, scale, eps, stream);
++    } else if (D <= 64) {
++        rope_pe_permute_rms_launch<dst_t, 64>(src0, pe, dst, n_rot, scale, eps, stream);
++    } else {
++        rope_pe_permute_rms_launch<dst_t, 128>(src0, pe, dst, n_rot, scale, eps, stream);
++    }
++}
++
+ template <typename dst_t>
+ static void rope_pe_permute_cuda(const ggml_tensor * src0, const float * pe, dst_t * dst, const int n_rot,
+                                  const float scale, cudaStream_t stream) {
+@@ -78,6 +184,17 @@ void ggml_cuda_op_rope_pe_permute(ggml_backend_cuda_context & ctx, ggml_tensor *
+     cudaStream_t  stream = ctx.stream();
+     const float * pe_d   = pe ? (const float *) pe->data : nullptr;
+ 
++    if (ggml_get_op_params_i32(dst, 2) != 0) {
++        const float eps = ggml_get_op_params_f32(dst, 3);
++        if (dst->type == GGML_TYPE_F16) {
++            rope_pe_permute_rms_cuda(src0, pe_d, (half *) dst->data, n_rot, scale, eps, stream);
++        } else {
++            GGML_ASSERT(dst->type == GGML_TYPE_F32);
++            rope_pe_permute_rms_cuda(src0, pe_d, (float *) dst->data, n_rot, scale, eps, stream);
++        }
++        return;
++    }
++
+     if (dst->type == GGML_TYPE_F16) {
+         rope_pe_permute_cuda(src0, pe_d, (half *) dst->data, n_rot, scale, stream);
+     } else {
+@@ -98,5 +215,8 @@ bool ggml_cuda_rope_pe_permute_supported(const ggml_tensor * op) {
+     if (src0->ne[2] > INT_MAX || src0->ne[3] > 65535 || src0->ne[1] > INT_MAX || src0->ne[0] > INT_MAX) {
+         return false;
+     }
++    if (ggml_get_op_params_i32(op, 2) != 0 && (src0->ne[0] > 128 || !rope_pe_rms_small_rows_enabled())) {
++        return false;
++    }
+     return pe == nullptr || (pe->type == GGML_TYPE_F32 && ggml_is_contiguous(pe));
+ }
+diff --git a/src/ggml.c b/src/ggml.c
+index a9f0b47d..1dcc8a0f 100644
+--- a/src/ggml.c
++++ b/src/ggml.c
+@@ -5758,6 +5758,21 @@ struct ggml_tensor * ggml_rope_pe_permute(
+     return result;
+ }
+ 
++struct ggml_tensor * ggml_rope_pe_permute_rms(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * a,
++        struct ggml_tensor  * pe,
++        int                   n_rot,
++        float                 scale,
++        enum ggml_type        type,
++        float                 eps) {
++    GGML_ASSERT(eps >= 0.0f);
++    struct ggml_tensor * result = ggml_rope_pe_permute(ctx, a, pe, n_rot, scale, type);
++    ggml_set_op_params_i32(result, 2, 1);
++    ggml_set_op_params_f32(result, 3, eps);
++    return result;
++}
++
+ static struct ggml_tensor * ggml_modulate_rows_impl(
+         struct ggml_context * ctx,
+         struct ggml_tensor  * acc,
+diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
+index fdda5dc7..108da4a2 100644
+--- a/tests/test-backend-ops.cpp
++++ b/tests/test-backend-ops.cpp
+@@ -2199,15 +2199,16 @@ struct test_rope_pe_permute : public test_case {
+     const int n_rot;
+     const float scale;
+     const bool view; // a is a strided view (one third of a fused qkv projection)
++    const float eps; // >= 0: ggml_rope_pe_permute_rms (rows RMS-normalised first)
+ 
+     std::string vars() override {
+-        return VARS_TO_STR5(type, ne_a, n_rot, scale, view);
++        return VARS_TO_STR6(type, ne_a, n_rot, scale, view, eps);
+     }
+ 
+     test_rope_pe_permute(ggml_type type = GGML_TYPE_F32,
+             std::array<int64_t, 4> ne_a = {128, 4, 9, 1},
+-            int n_rot = 96, float scale = 1.0f, bool view = false)
+-        : type(type), ne_a(ne_a), n_rot(n_rot), scale(scale), view(view) {}
++            int n_rot = 96, float scale = 1.0f, bool view = false, float eps = -1.0f)
++        : type(type), ne_a(ne_a), n_rot(n_rot), scale(scale), view(view), eps(eps) {}
+ 
+     ggml_tensor * build_graph(ggml_context * ctx) override {
+         ggml_tensor * a;
+@@ -2226,7 +2227,8 @@ struct test_rope_pe_permute : public test_case {
+             pe = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 2, 2, n_rot / 2, ne_a[2]);
+             ggml_set_name(pe, "pe");
+         }
+-        ggml_tensor * out = ggml_rope_pe_permute(ctx, a, pe, n_rot, scale, type);
++        ggml_tensor * out = eps >= 0.0f ? ggml_rope_pe_permute_rms(ctx, a, pe, n_rot, scale, type, eps)
++                                        : ggml_rope_pe_permute(ctx, a, pe, n_rot, scale, type);
+         ggml_set_name(out, "out");
+         return out;
+     }
+@@ -9879,6 +9881,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
+                 test_cases.emplace_back(new test_rope_pe_permute(type, {128, 4, 9, 1}, 0, scale, view));
+                 test_cases.emplace_back(new test_rope_pe_permute(type, {64, 3, 7, 2}, 32, scale, view));
+                 test_cases.emplace_back(new test_rope_pe_permute(type, {200, 2, 3, 1}, 100, scale, view));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {64, 32, 37, 1}, 48, scale, view, 1e-5f));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {64, 3, 7, 2}, 0, scale, view, 1e-5f));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {128, 4, 9, 1}, 96, scale, view, 1e-6f));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {32, 2, 5, 1}, 16, scale, view, 1e-5f));
++                test_cases.emplace_back(new test_rope_pe_permute(type, {48, 2, 5, 1}, 32, scale, view, 1e-5f));
+             }
+         }
+     }
diff --git a/scripts/unsloth/ggml-patches/0006-ggml-cuda-cudnn-attention.patch b/scripts/unsloth/ggml-patches/0006-ggml-cuda-cudnn-attention.patch
new file mode 100644
index 0000000000..73fb6cd4cc
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0006-ggml-cuda-cudnn-attention.patch
@@ -0,0 +1,994 @@
+From 1602a3095b12f20b584066ac88c37a9b2ad07f1d Mon Sep 17 00:00:00 2001
+From: Daniel Han <danielhanchen@gmail.com>
+Date: Sat, 3 Oct 2026 18:27:29 -0700
+Subject: [PATCH] ggml-cuda: optional cuDNN attention for unmasked
+ FLASH_ATTN_EXT and SAGE_ATTN
+
+GGML_CUDA_CUDNN=ON (default OFF, Linux) builds a cuDNN fused-attention path
+through the header-only cudnn-frontend v1.26.0 (MIT; fetched by CMake or taken
+from GGML_CUDA_CUDNN_FRONTEND_DIR) with NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING:
+only cudnn.h is needed at build time and libcudnn.so.9 is dlopen'ed on first use
+(GGML_CUDA_CUDNN_LIB overrides the name), so the binary has no cuDNN link
+dependency and runs unchanged where cuDNN is missing.
+
+On NVIDIA sm80+ it can take unmasked, sink/ALiBi/softcap-free attention with equal
+Q/K/V head sizes (multiple of 8, up to 256) and at least 2^20 attention scores,
+GQA and batch included, for FLASH_ATTN_EXT and for SAGE_ATTN in AUTO mode (an
+explicitly requested Sage variant keeps its kernel). Q (F32) is narrowed to F16
+into dst's own bytes, F16 K/V are read in place with their strides (F32 K/V are
+narrowed), cuDNN writes F16 O to scratch after dst and one kernel widens it into
+dst. The scratch is part of the op's graph allocation (max with the existing
+kernels' needs). Plans are built once per shape and cached; a shape cuDNN
+declines, a failed build or execute, stream capture, Turing and older, or a
+missing library keep the existing kernels.
+
+By default FLASH_ATTN_EXT takes it on sm89 and sm100 and SAGE_ATTN on sm100,
+where it measured faster; the existing kernels were faster on sm86 (both ops)
+and sm89 (Sage), and other architectures were not measured.
+GGML_CUDA_CUDNN_ATTN=0|1 and GGML_CUDA_CUDNN_SAGE=0|1 turn it off or on for every
+sm80+ GPU, GGML_CUDA_CUDNN_ATTN_BF16=1 runs BF16, GGML_CUDA_CUDNN_ATTN_LOG=1
+logs each call.
+
+test-backend-ops: unmasked cases at the MiniMax-H3 head layout (odd and aligned
+lengths, GQA with batch, F32 K/V, permuted Q) and unmasked perf cases.
+---
+ CMakeLists.txt                     |   3 +
+ src/ggml-cuda/CMakeLists.txt       |  49 ++++
+ src/ggml-cuda/cudnn-attn-graph.cpp | 258 +++++++++++++++++++
+ src/ggml-cuda/cudnn-attn-graph.h   |  31 +++
+ src/ggml-cuda/cudnn-attn.cu        | 384 +++++++++++++++++++++++++++++
+ src/ggml-cuda/cudnn-attn.cuh       |  26 ++
+ src/ggml-cuda/fattn.cu             |   7 +-
+ src/ggml-cuda/ggml-cuda.cu         |   2 +-
+ src/ggml-cuda/sage-attn.cu         |  12 +-
+ src/ggml-cuda/sage-attn.cuh        |   2 +-
+ tests/test-backend-ops.cpp         |  25 ++
+ 11 files changed, 793 insertions(+), 6 deletions(-)
+ create mode 100644 src/ggml-cuda/cudnn-attn-graph.cpp
+ create mode 100644 src/ggml-cuda/cudnn-attn-graph.h
+ create mode 100644 src/ggml-cuda/cudnn-attn.cu
+ create mode 100644 src/ggml-cuda/cudnn-attn.cuh
+
+[Ported onto leejet/ggml dc998771 (stable-diffusion.cpp master-982): the diff below is
+ the whole patch as one diff, regenerated from the port.]
+---
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 35252a83..da110ce1 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -209,6 +209,9 @@ set   (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING
+                                             "ggml: FlashAttention K-V type combinations to compile, \"all\" or a list such as \"q8_0-q8_0;q8_0-q4_0\"")
+ option(GGML_CUDA_GRAPHS                     "ggml: use CUDA graphs (llama.cpp only)"          ${GGML_CUDA_GRAPHS_DEFAULT})
+ option(GGML_CUDA_NCCL                       "ggml: use NVIDIA Collective Comm. Library"       ON)
++option(GGML_CUDA_CUDNN                      "ggml: cuDNN attention (cudnn-frontend, libcudnn opened at run time)" OFF)
++set   (GGML_CUDA_CUDNN_FRONTEND_DIR "" CACHE PATH "ggml: cudnn-frontend checkout for GGML_CUDA_CUDNN (fetched when empty)")
++set   (GGML_CUDA_CUDNN_INCLUDE_DIR  "" CACHE PATH "ggml: directory holding cudnn.h for GGML_CUDA_CUDNN")
+ set   (GGML_CUDA_COMPRESSION_MODE "size" CACHE STRING
+                                             "ggml: cuda link binary compression mode; requires cuda 12.8+")
+ set_property(CACHE GGML_CUDA_COMPRESSION_MODE PROPERTY STRINGS "none;speed;balance;size")
+diff --git a/src/ggml-cuda/CMakeLists.txt b/src/ggml-cuda/CMakeLists.txt
+index 37aa6d94..7c2fb7dd 100644
+--- a/src/ggml-cuda/CMakeLists.txt
++++ b/src/ggml-cuda/CMakeLists.txt
+@@ -127,11 +127,60 @@ if (CUDAToolkit_FOUND)
+     ggml_cuda_fattn_vec_instances(${CMAKE_CURRENT_SOURCE_DIR} SRCS)
+     list(APPEND GGML_SOURCES_CUDA ${SRCS})
+ 
++    if (GGML_CUDA_CUDNN AND WIN32)
++        message(WARNING "GGML_CUDA_CUDNN: the run-time cuDNN loader is Linux only, building without it")
++        set(GGML_CUDA_CUDNN OFF)
++    endif()
++    if (GGML_CUDA_CUDNN)
++        # Headers only at build time: cudnn.h for the types, cudnn-frontend for the graph API.
++        # libcudnn itself is opened with dlopen when the first attention op runs.
++        if (GGML_CUDA_CUDNN_INCLUDE_DIR)
++            # an explicit directory wins over any earlier cached search result
++            set(GGML_CUDNN_HEADER_DIR ${GGML_CUDA_CUDNN_INCLUDE_DIR})
++        else()
++            find_path(GGML_CUDNN_HEADER_DIR cudnn.h
++                      HINTS $ENV{CUDNN_INCLUDE_DIR} $ENV{CUDNN_PATH} ${CUDAToolkit_INCLUDE_DIRS}
++                      PATH_SUFFIXES include)
++        endif()
++        if (NOT GGML_CUDNN_HEADER_DIR OR NOT EXISTS ${GGML_CUDNN_HEADER_DIR}/cudnn.h)
++            message(FATAL_ERROR "GGML_CUDA_CUDNN: cudnn.h not found, set GGML_CUDA_CUDNN_INCLUDE_DIR")
++        endif()
++        if (GGML_CUDA_CUDNN_FRONTEND_DIR)
++            set(GGML_CUDNN_FRONTEND_INCLUDE ${GGML_CUDA_CUDNN_FRONTEND_DIR}/include)
++        else()
++            include(FetchContent)
++            # v1.26.0, the last MIT-licensed release (later ones are Apache-2.0). Only the headers are
++            # used; SOURCE_SUBDIR points at a directory that does not exist so its CMake project stays out.
++            FetchContent_Declare(
++                ggml_cudnn_frontend
++                URL      https://github.com/NVIDIA/cudnn-frontend/archive/refs/tags/v1.26.0.tar.gz
++                URL_HASH SHA256=f88361fb714589c970bac2ab56d6a7843722be08ca5ba226484de3fa46fd22cf
++                DOWNLOAD_EXTRACT_TIMESTAMP TRUE
++                SOURCE_SUBDIR ggml-headers-only
++            )
++            FetchContent_MakeAvailable(ggml_cudnn_frontend)
++            set(GGML_CUDNN_FRONTEND_INCLUDE ${ggml_cudnn_frontend_SOURCE_DIR}/include)
++        endif()
++        if (NOT EXISTS ${GGML_CUDNN_FRONTEND_INCLUDE}/cudnn_frontend.h)
++            message(FATAL_ERROR "GGML_CUDA_CUDNN: cudnn_frontend.h not found under ${GGML_CUDNN_FRONTEND_INCLUDE}")
++        endif()
++        message(STATUS "cuDNN attention: cudnn.h in ${GGML_CUDNN_HEADER_DIR}, cudnn-frontend in ${GGML_CUDNN_FRONTEND_INCLUDE}")
++        list(APPEND GGML_SOURCES_CUDA cudnn-attn-graph.cpp)
++    endif()
++
+     ggml_add_backend_library(ggml-cuda
+                              ${GGML_HEADERS_CUDA}
+                              ${GGML_SOURCES_CUDA}
+                             )
+ 
++    if (GGML_CUDA_CUDNN)
++        target_compile_definitions(ggml-cuda PRIVATE GGML_CUDA_USE_CUDNN NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING)
++        # -isystem: third-party headers, kept out of ggml's warning flags.
++        set_source_files_properties(cudnn-attn-graph.cpp PROPERTIES
++            COMPILE_OPTIONS "-isystem;${GGML_CUDNN_FRONTEND_INCLUDE};-isystem;${GGML_CUDNN_HEADER_DIR}")
++        target_link_libraries(ggml-cuda PRIVATE ${CMAKE_DL_LIBS})
++    endif()
++
+     if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "11.8")
+         target_compile_definitions(ggml-cuda PRIVATE GGML_CUDA_USE_CUBLASLT_FP8)
+         if (NOT GGML_STATIC OR WIN32)
+diff --git a/src/ggml-cuda/cudnn-attn-graph.cpp b/src/ggml-cuda/cudnn-attn-graph.cpp
+new file mode 100644
+index 00000000..b0f1dc11
+--- /dev/null
++++ b/src/ggml-cuda/cudnn-attn-graph.cpp
+@@ -0,0 +1,258 @@
++// cuDNN scaled dot product attention through cudnn-frontend (header-only), with libcudnn opened at
++// run time (NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING): a build with GGML_CUDA_CUDNN still starts and
++// runs on hosts without cuDNN, it just keeps the ggml kernels.
++
++#include "cudnn-attn-graph.h"
++
++#include "ggml-impl.h"
++
++#include <cudnn_frontend.h>
++
++#include <dlfcn.h>
++
++#include <array>
++#include <chrono>
++#include <cstdio>
++#include <cstdlib>
++#include <cstring>
++#include <map>
++#include <memory>
++#include <mutex>
++#include <string>
++
++namespace cudnn_frontend {
++// cudnn-frontend resolves every backend call through this handle when dynamic loading is on.
++void * cudnn_dlhandle = nullptr;
++}  // namespace cudnn_frontend
++
++namespace fe = cudnn_frontend;
++
++namespace {
++
++enum : int64_t { UID_Q = 1, UID_K = 2, UID_V = 3, UID_O = 4 };
++
++using plan_key = std::array<int64_t, 25>;
++
++struct plan_entry {
++    std::shared_ptr<fe::graph::Graph> graph;  // null: cuDNN has no plan for this key
++    int64_t                           workspace = 0;
++};
++
++std::mutex                    g_mutex;
++int                           g_load_state = 0;  // 0 not tried, 1 loaded, -1 unavailable
++std::map<int, cudnnHandle_t>  g_handles;
++std::map<plan_key, plan_entry> g_plans;
++
++plan_key make_key(const ggml_cuda_cudnn_sdpa_params & p) {
++    plan_key k{};
++    int      i = 0;
++    k[i++]     = p.device;
++    k[i++]     = p.b;
++    k[i++]     = p.hq;
++    k[i++]     = p.hk;
++    k[i++]     = p.sq;
++    k[i++]     = p.skv;
++    k[i++]     = p.d;
++    for (int j = 0; j < 4; ++j) {
++        k[i++] = p.q_stride[j];
++    }
++    for (int j = 0; j < 4; ++j) {
++        k[i++] = p.k_stride[j];
++    }
++    for (int j = 0; j < 4; ++j) {
++        k[i++] = p.v_stride[j];
++    }
++    for (int j = 0; j < 4; ++j) {
++        k[i++] = p.o_stride[j];
++    }
++    uint32_t scale_bits = 0;
++    std::memcpy(&scale_bits, &p.scale, sizeof(scale_bits));
++    k[i++] = scale_bits;
++    k[i++] = p.bf16 ? 1 : 0;
++    GGML_ASSERT(i == (int) k.size());
++    return k;
++}
++
++bool load_locked() {
++    if (g_load_state != 0) {
++        return g_load_state == 1;
++    }
++    g_load_state = -1;
++
++    // cudnn-frontend opens libcudart itself for a few runtime calls; point it at the runtime major
++    // this backend was built against so it never picks a second, mismatched copy.
++    char cudart_name[32];
++    std::snprintf(cudart_name, sizeof(cudart_name), "libcudart.so.%d", CUDART_VERSION / 1000);
++    setenv("CUDNN_FRONTEND_CUDART_LIB_NAME", cudart_name, 0);
++
++    const char * user = std::getenv("GGML_CUDA_CUDNN_LIB");
++    const char * name = user != nullptr && user[0] != '\0' ? user : "libcudnn.so.9";
++    void *       h    = dlopen(name, RTLD_NOW | RTLD_LOCAL);
++    if (h == nullptr) {
++        GGML_LOG_DEBUG("%s: cuDNN not loaded (%s), attention keeps the ggml kernels\n", __func__, dlerror());
++        return false;
++    }
++    using get_version_t = size_t (*)();
++    auto get_version    = reinterpret_cast<get_version_t>(dlsym(h, "cudnnGetVersion"));
++    const size_t version = get_version != nullptr ? get_version() : 0;
++    if (version < 90000) {
++        GGML_LOG_WARN("%s: %s reports cuDNN %zu, need 9.0 or newer; attention keeps the ggml kernels\n", __func__,
++                      name, version);
++        dlclose(h);
++        return false;
++    }
++    fe::cudnn_dlhandle = h;
++    g_load_state       = 1;
++    GGML_LOG_INFO("%s: cuDNN %zu loaded from %s for attention\n", __func__, version, name);
++    return true;
++}
++
++cudnnHandle_t handle_locked(int device) {
++    auto it = g_handles.find(device);
++    if (it != g_handles.end()) {
++        return it->second;
++    }
++    cudnnHandle_t h = nullptr;
++    if (fe::detail::create_handle(&h) != CUDNN_STATUS_SUCCESS) {
++        h = nullptr;
++    }
++    g_handles[device] = h;
++    return h;
++}
++
++std::shared_ptr<fe::graph::Graph> build_graph(const ggml_cuda_cudnn_sdpa_params & p, cudnnHandle_t handle,
++                                              std::string & why) {
++    auto g = std::make_shared<fe::graph::Graph>();
++    g->set_io_data_type(p.bf16 ? fe::DataType_t::BFLOAT16 : fe::DataType_t::HALF)
++        .set_intermediate_data_type(fe::DataType_t::FLOAT)
++        .set_compute_data_type(fe::DataType_t::FLOAT);
++
++    auto tensor = [&](const char * name, int64_t uid, int64_t h, int64_t s, const int64_t * stride) {
++        return g->tensor(fe::graph::Tensor_attributes()
++                             .set_name(name)
++                             .set_uid(uid)
++                             .set_dim({ p.b, h, s, p.d })
++                             .set_stride({ stride[0], stride[1], stride[2], stride[3] }));
++    };
++    auto Q = tensor("Q", UID_Q, p.hq, p.sq, p.q_stride);
++    auto K = tensor("K", UID_K, p.hk, p.skv, p.k_stride);
++    auto V = tensor("V", UID_V, p.hk, p.skv, p.v_stride);
++
++    auto options = fe::graph::SDPA_attributes().set_name("ggml_sdpa").set_generate_stats(false).set_attn_scale(p.scale);
++    auto [O, stats] = g->sdpa(Q, K, V, options);
++    (void) stats;
++    O->set_output(true)
++        .set_uid(UID_O)
++        .set_dim({ p.b, p.hq, p.sq, p.d })
++        .set_stride({ p.o_stride[0], p.o_stride[1], p.o_stride[2], p.o_stride[3] });
++
++    fe::error_t st = g->validate();
++    if (st.is_good()) {
++        st = g->build_operation_graph(handle);
++    }
++    if (st.is_good()) {
++        st = g->create_execution_plans({ fe::HeurMode_t::A, fe::HeurMode_t::FALLBACK });
++    }
++    if (st.is_good()) {
++        st = g->check_support(handle);
++    }
++    if (st.is_good()) {
++        st = g->build_plans(handle);
++    }
++    if (!st.is_good()) {
++        why = st.get_message();
++        return nullptr;
++    }
++    return g;
++}
++
++}  // namespace
++
++bool ggml_cuda_cudnn_available() {
++    std::lock_guard<std::mutex> lock(g_mutex);
++    try {
++        return load_locked();
++    } catch (const std::exception & e) {
++        GGML_LOG_WARN("%s: %s\n", __func__, e.what());
++        g_load_state = -1;
++        return false;
++    }
++}
++
++bool ggml_cuda_cudnn_sdpa_prepare(const ggml_cuda_cudnn_sdpa_params & p, size_t * workspace_size) {
++    std::lock_guard<std::mutex> lock(g_mutex);
++    if (g_load_state != 1) {
++        return false;
++    }
++    const plan_key key = make_key(p);
++    auto           it  = g_plans.find(key);
++    if (it == g_plans.end()) {
++        plan_entry  entry;
++        std::string why;
++        const auto  t0 = std::chrono::steady_clock::now();
++        try {
++            cudnnHandle_t handle = handle_locked(p.device);
++            if (handle == nullptr) {
++                why = "cudnnCreate failed";
++            } else {
++                entry.graph = build_graph(p, handle, why);
++                if (entry.graph && !entry.graph->get_workspace_size(entry.workspace).is_good()) {
++                    entry.graph = nullptr;
++                    why         = "workspace query failed";
++                }
++            }
++        } catch (const std::exception & e) {
++            entry.graph = nullptr;
++            why         = e.what();
++        }
++        const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
++        if (entry.graph) {
++            GGML_LOG_INFO("%s: cuDNN SDPA plan b=%lld hq=%lld hk=%lld sq=%lld skv=%lld d=%lld %s, workspace %lld B, "
++                          "built in %.0f ms\n",
++                          __func__, (long long) p.b, (long long) p.hq, (long long) p.hk, (long long) p.sq,
++                          (long long) p.skv, (long long) p.d, p.bf16 ? "bf16" : "f16", (long long) entry.workspace,
++                          ms);
++        } else {
++            GGML_LOG_INFO("%s: no cuDNN SDPA plan for b=%lld hq=%lld hk=%lld sq=%lld skv=%lld d=%lld, keeping the ggml "
++                          "kernel (%s)\n",
++                          __func__, (long long) p.b, (long long) p.hq, (long long) p.hk, (long long) p.sq,
++                          (long long) p.skv, (long long) p.d, why.c_str());
++        }
++        it = g_plans.emplace(key, std::move(entry)).first;
++    }
++    if (!it->second.graph) {
++        return false;
++    }
++    *workspace_size = (size_t) it->second.workspace;
++    return true;
++}
++
++bool ggml_cuda_cudnn_sdpa_execute(const ggml_cuda_cudnn_sdpa_params & p, cudaStream_t stream, void * q, void * k,
++                                  void * v, void * o, void * workspace) {
++    std::lock_guard<std::mutex> lock(g_mutex);
++    auto                        it = g_plans.find(make_key(p));
++    if (it == g_plans.end() || !it->second.graph) {
++        return false;
++    }
++    try {
++        cudnnHandle_t handle = handle_locked(p.device);
++        if (handle == nullptr || fe::detail::set_stream(handle, stream) != CUDNN_STATUS_SUCCESS) {
++            return false;
++        }
++        std::unordered_map<fe::graph::Tensor_attributes::uid_t, void *> pack = {
++            { UID_Q, q }, { UID_K, k }, { UID_V, v }, { UID_O, o }
++        };
++        fe::error_t st = it->second.graph->execute(handle, pack, workspace);
++        if (!st.is_good()) {
++            GGML_LOG_WARN("%s: cuDNN SDPA execute failed, keeping the ggml kernel for this shape: %s\n", __func__,
++                          st.get_message().c_str());
++            it->second.graph = nullptr;
++            return false;
++        }
++    } catch (const std::exception & e) {
++        GGML_LOG_WARN("%s: cuDNN SDPA execute failed, keeping the ggml kernel for this shape: %s\n", __func__, e.what());
++        it->second.graph = nullptr;
++        return false;
++    }
++    return true;
++}
+diff --git a/src/ggml-cuda/cudnn-attn-graph.h b/src/ggml-cuda/cudnn-attn-graph.h
+new file mode 100644
+index 00000000..9749f2ab
+--- /dev/null
++++ b/src/ggml-cuda/cudnn-attn-graph.h
+@@ -0,0 +1,31 @@
++#pragma once
++
++// Host-side half of the cuDNN attention path (cudnn-attn.cu is the device side). Only compiled
++// with GGML_CUDA_CUDNN. libcudnn is opened at run time, so the binary never links against it.
++
++#include <cstddef>
++#include <cstdint>
++
++#include <cuda_runtime.h>
++
++struct ggml_cuda_cudnn_sdpa_params {
++    int     device;
++    int64_t b, hq, hk, sq, skv, d;  // batch, query heads, key/value heads, query tokens, key tokens, head size
++    int64_t q_stride[4];            // element strides in (b, h, s, d) order
++    int64_t k_stride[4];
++    int64_t v_stride[4];
++    int64_t o_stride[4];
++    float   scale;
++    bool    bf16;                   // io type: BF16 when set, F16 otherwise
++};
++
++// Opens libcudnn once per process. False when it cannot be loaded or is older than cuDNN 9.
++bool ggml_cuda_cudnn_available();
++
++// Builds (or finds) the execution plan for these parameters. False when cuDNN has no plan for
++// them; the result is cached either way, so a rejected shape costs one attempt per process.
++bool ggml_cuda_cudnn_sdpa_prepare(const ggml_cuda_cudnn_sdpa_params & p, size_t * workspace_size);
++
++// Runs a plan built by ggml_cuda_cudnn_sdpa_prepare on the given stream.
++bool ggml_cuda_cudnn_sdpa_execute(const ggml_cuda_cudnn_sdpa_params & p, cudaStream_t stream,
++                                  void * q, void * k, void * v, void * o, void * workspace);
+diff --git a/src/ggml-cuda/cudnn-attn.cu b/src/ggml-cuda/cudnn-attn.cu
+new file mode 100644
+index 00000000..a5b9de14
+--- /dev/null
++++ b/src/ggml-cuda/cudnn-attn.cu
+@@ -0,0 +1,384 @@
++#include "cudnn-attn.cuh"
++
++#if defined(GGML_CUDA_USE_CUDNN) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
++
++#include "convert.cuh"
++#include "cudnn-attn-graph.h"
++
++#include <atomic>
++#include <cstdlib>
++#include <cstring>
++
++static bool cudnn_attn_env_set(const char * name, bool default_value) {
++    const char * v = std::getenv(name);
++    if (v == nullptr || v[0] == '\0') {
++        return default_value;
++    }
++    return !(std::strcmp(v, "0") == 0 || std::strcmp(v, "off") == 0 || std::strcmp(v, "false") == 0);
++}
++
++// Where a staged F16 / BF16 tensor lives: inside dst's bytes or in the scratch after dst.
++struct cudnn_attn_slot {
++    bool   in_dst;
++    size_t off;
++};
++
++// Everything the cuDNN path needs to know about one op, from shapes and types alone.
++struct cudnn_attn_plan {
++    ggml_cuda_cudnn_sdpa_params p;
++    bool                        convert_k;
++    bool                        convert_v;
++    cudnn_attn_slot             q, k, v, o;
++    size_t                      scratch;  // scratch bytes after dst
++};
++
++static bool cudnn_attn_strides_ok(const ggml_tensor * t) {
++    // cuDNN wants a unit stride on the head dimension and 16-byte aligned rows.
++    return t->nb[0] == ggml_type_size(t->type) && t->nb[1] % 16 == 0 && t->nb[2] % 16 == 0 && t->nb[3] % 16 == 0;
++}
++
++// -1 when unset, otherwise 0 or 1.
++static int cudnn_attn_env_mode(const char * name) {
++    const char * v = std::getenv(name);
++    if (v == nullptr || v[0] == '\0') {
++        return -1;
++    }
++    return cudnn_attn_env_set(name, true) ? 1 : 0;
++}
++
++static bool cudnn_attn_make_plan(int device, const ggml_tensor * dst, cudnn_attn_plan & plan) {
++    static const int  attn_mode = cudnn_attn_env_mode("GGML_CUDA_CUDNN_ATTN");
++    static const int  sage_mode = cudnn_attn_env_mode("GGML_CUDA_CUDNN_SAGE");
++    static const bool want_bf16 = cudnn_attn_env_set("GGML_CUDA_CUDNN_ATTN_BF16", false);
++    if (attn_mode == 0 || (dst->op == GGML_OP_SAGE_ATTN && sage_mode == 0)) {
++        return false;
++    }
++    if (dst->op != GGML_OP_FLASH_ATTN_EXT && dst->op != GGML_OP_SAGE_ATTN) {
++        return false;
++    }
++    // An explicitly requested SageAttention variant keeps its kernel; AUTO takes the cuDNN path.
++    if (dst->op == GGML_OP_SAGE_ATTN && ggml_get_op_params_i32(dst, 1) != GGML_SAGE_ATTN_AUTO) {
++        return false;
++    }
++    const int cc = ggml_cuda_info().devices[device].cc;
++    if (!GGML_CUDA_CC_IS_NVIDIA(cc) || cc < GGML_CUDA_CC_AMPERE) {
++        return false;
++    }
++    // Unset switches keep the existing kernels except where cuDNN measured faster at H3 shapes: against the flash
++    // attention kernel on sm89 and sm100 (slower on sm86), against the Sage kernel on sm100 (slower on sm86 and
++    // sm89). Other architectures were not measured. =1 opts in on any sm80+ GPU.
++    const bool sm100 = cc >= 1000 && cc < GGML_CUDA_CC_BLACKWELL;
++    const bool sm89  = cc >= GGML_CUDA_CC_ADA_LOVELACE && cc < GGML_CUDA_CC_HOPPER;
++    if (dst->op == GGML_OP_SAGE_ATTN ? sage_mode < 0 && !sm100 : attn_mode < 0 && !sm89 && !sm100) {
++        return false;
++    }
++
++    const ggml_tensor * Q = dst->src[0];
++    const ggml_tensor * K = dst->src[1];
++    const ggml_tensor * V = dst->src[2];
++    if (Q == nullptr || K == nullptr || V == nullptr) {
++        return false;
++    }
++    if (dst->op == GGML_OP_FLASH_ATTN_EXT) {
++        float max_bias = 0.0f, softcap = 0.0f;
++        std::memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));
++        std::memcpy(&softcap, (const float *) dst->op_params + 2, sizeof(float));
++        if (dst->src[3] != nullptr || dst->src[4] != nullptr || max_bias != 0.0f || softcap != 0.0f) {
++            return false;
++        }
++    }
++
++    const ggml_type io = want_bf16 ? GGML_TYPE_BF16 : GGML_TYPE_F16;
++    auto kv_type_ok    = [](ggml_type t) {
++        return t == GGML_TYPE_F16 || t == GGML_TYPE_F32 || t == GGML_TYPE_BF16;
++    };
++    if (Q->type != GGML_TYPE_F32 || !kv_type_ok(K->type) || !kv_type_ok(V->type) || dst->type != GGML_TYPE_F32) {
++        return false;
++    }
++    // BF16 -> F16 can overflow, so BF16 K/V only run in BF16 mode. F16 K/V in BF16 mode are narrowed
++    // (3 mantissa bits lost); that mode exists to measure BF16 against the F16 default.
++    if ((K->type == GGML_TYPE_BF16 || V->type == GGML_TYPE_BF16) && io != GGML_TYPE_BF16) {
++        return false;
++    }
++
++    const int64_t d = Q->ne[0];
++    if (d % 8 != 0 || d > 256 || K->ne[0] != d || V->ne[0] != d) {
++        return false;
++    }
++    const int64_t sq = Q->ne[1], hq = Q->ne[2], b = Q->ne[3];
++    const int64_t skv = K->ne[1], hk = K->ne[2];
++    if (V->ne[1] != skv || V->ne[2] != hk || K->ne[3] != b || V->ne[3] != b || hk == 0 || hq % hk != 0) {
++        return false;
++    }
++    if (!ggml_is_contiguous(dst) || dst->ne[0] != d || dst->ne[1] != hq || dst->ne[2] != sq || dst->ne[3] != b) {
++        return false;
++    }
++    if (Q->nb[0] != ggml_type_size(Q->type) || !cudnn_attn_strides_ok(K) || !cudnn_attn_strides_ok(V)) {
++        return false;
++    }
++    if (hq * b > INT32_MAX || sq > INT32_MAX || skv > INT32_MAX) {
++        return false;
++    }
++    // Below about 2^20 attention scores the two extra conversion launches cost more than cuDNN saves
++    // (B200: 31 tokens x 56 heads 10.6 -> 14-16 us, one query row x 32 heads x 4096 keys 12.9 -> 20 us,
++    // break-even near 256 x 256 x 16, 2.3x faster at 512 x 512 x 16), so small shapes keep the ggml kernels.
++    if ((double) sq * (double) skv * (double) (hq * b) < (double) (1 << 20)) {
++        return false;
++    }
++    if (!ggml_cuda_cudnn_available()) {
++        return false;
++    }
++
++    plan               = {};
++    plan.p.device      = device;
++    plan.p.b           = b;
++    plan.p.hq          = hq;
++    plan.p.hk          = hk;
++    plan.p.sq          = sq;
++    plan.p.skv         = skv;
++    plan.p.d           = d;
++    plan.p.bf16        = io == GGML_TYPE_BF16;
++    std::memcpy(&plan.p.scale, (const float *) dst->op_params + 0, sizeof(float));
++
++    // Converted tensors are written contiguous: [d, s, h, b], i.e. (b, h, s, d) strides below.
++    auto set_contiguous = [d](int64_t * stride, int64_t s, int64_t h) {
++        stride[3] = 1;
++        stride[2] = d;
++        stride[1] = d * s;
++        stride[0] = d * s * h;
++    };
++    auto set_from = [](int64_t * stride, const ggml_tensor * t) {
++        const size_t ts = ggml_type_size(t->type);
++        stride[3]       = 1;
++        stride[2]       = t->nb[1] / ts;
++        stride[1]       = t->nb[2] / ts;
++        stride[0]       = t->nb[3] / ts;
++    };
++
++    // Q (and K/V when they need converting) are staged in dst's own bytes: dst is F32 and at least
++    // twice the size of F16 Q, and it is only written after cuDNN has read them. O and whatever does
++    // not fit go to scratch after dst.
++    const size_t es       = 2;  // F16 / BF16
++    const size_t dst_size = ggml_nbytes(dst);
++    size_t       dst_off  = 0;
++    size_t       off      = 0;
++    auto         take     = [&](size_t bytes, cudnn_attn_slot & slot, bool allow_dst) {
++        if (allow_dst && dst_off + bytes <= dst_size) {
++            slot = { true, dst_off };
++            dst_off += GGML_PAD(bytes, 256);
++        } else {
++            slot = { false, off };
++            off += GGML_PAD(bytes, 256);
++        }
++    };
++
++    take(ggml_nelements(Q) * es, plan.q, true);
++    set_contiguous(plan.p.q_stride, sq, hq);
++
++    plan.convert_k = K->type != io;
++    if (plan.convert_k) {
++        take(ggml_nelements(K) * es, plan.k, true);
++        set_contiguous(plan.p.k_stride, skv, hk);
++    } else {
++        set_from(plan.p.k_stride, K);
++    }
++    plan.convert_v = V->type != io;
++    if (plan.convert_v) {
++        take(ggml_nelements(V) * es, plan.v, true);
++        set_contiguous(plan.p.v_stride, skv, hk);
++    } else {
++        set_from(plan.p.v_stride, V);
++    }
++
++    // O is written in dst's layout, [d, hq, sq, b], then widened to F32 into dst.
++    take(ggml_nelements(dst) * es, plan.o, false);
++    plan.p.o_stride[3] = 1;
++    plan.p.o_stride[1] = d;
++    plan.p.o_stride[2] = d * hq;
++    plan.p.o_stride[0] = d * hq * sq;
++    plan.scratch       = off;
++    return true;
++}
++
++size_t ggml_cuda_cudnn_attn_alloc_size(int device, const ggml_tensor * dst) {
++    cudnn_attn_plan plan;
++    if (!cudnn_attn_make_plan(device, dst, plan)) {
++        return 0;
++    }
++    // dst (which also stages Q), up to 256 bytes to align the scratch start, then the scratch.
++    return ggml_nbytes(dst) + 256 + plan.scratch;
++}
++
++// F32 rows of 8-element groups (strided rows, 16-byte aligned) -> contiguous F16 / BF16, 8 per thread.
++template <typename dst_t>
++static __global__ void cudnn_attn_narrow_f32(const char * __restrict__ x, dst_t * __restrict__ y, const int64_t n8,
++                                             const int64_t g0, const int64_t ne1, const int64_t ne2, const int64_t nb1,
++                                             const int64_t nb2, const int64_t nb3) {
++    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n8; i += (int64_t) gridDim.x * blockDim.x) {
++        const int64_t g   = i % g0;
++        const int64_t row = i / g0;
++        const int64_t i1  = row % ne1;
++        const int64_t i2  = (row / ne1) % ne2;
++        const int64_t i3  = row / (ne1 * ne2);
++        const float4 * src = (const float4 *) (x + i3 * nb3 + i2 * nb2 + i1 * nb1) + 2 * g;
++        const float4   a   = src[0];
++        const float4   b   = src[1];
++        union {
++            dst_t h[8];
++            uint4 u;
++        } out;
++        out.h[0] = ggml_cuda_cast<dst_t>(a.x);
++        out.h[1] = ggml_cuda_cast<dst_t>(a.y);
++        out.h[2] = ggml_cuda_cast<dst_t>(a.z);
++        out.h[3] = ggml_cuda_cast<dst_t>(a.w);
++        out.h[4] = ggml_cuda_cast<dst_t>(b.x);
++        out.h[5] = ggml_cuda_cast<dst_t>(b.y);
++        out.h[6] = ggml_cuda_cast<dst_t>(b.z);
++        out.h[7] = ggml_cuda_cast<dst_t>(b.w);
++        ((uint4 *) y)[i] = out.u;
++    }
++}
++
++// Contiguous F16 / BF16 -> contiguous F32, 8 per thread.
++template <typename src_t>
++static __global__ void cudnn_attn_widen(const src_t * __restrict__ x, float * __restrict__ y, const int64_t n8) {
++    for (int64_t i = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; i < n8; i += (int64_t) gridDim.x * blockDim.x) {
++        union {
++            src_t h[8];
++            uint4 u;
++        } in;
++        in.u = ((const uint4 *) x)[i];
++        float4 a, b;
++        a.x = ggml_cuda_cast<float>(in.h[0]);
++        a.y = ggml_cuda_cast<float>(in.h[1]);
++        a.z = ggml_cuda_cast<float>(in.h[2]);
++        a.w = ggml_cuda_cast<float>(in.h[3]);
++        b.x = ggml_cuda_cast<float>(in.h[4]);
++        b.y = ggml_cuda_cast<float>(in.h[5]);
++        b.z = ggml_cuda_cast<float>(in.h[6]);
++        b.w = ggml_cuda_cast<float>(in.h[7]);
++        ((float4 *) y)[2 * i]     = a;
++        ((float4 *) y)[2 * i + 1] = b;
++    }
++}
++
++static int cudnn_attn_grid(int64_t n) {
++    return (int) std::min<int64_t>((n + 255) / 256, 65535LL * 8);
++}
++
++static void cudnn_attn_convert(const ggml_tensor * src, void * dst, bool bf16, cudaStream_t stream) {
++    const size_t ts = ggml_type_size(src->type);
++    const bool   vec_ok = src->type == GGML_TYPE_F32 && src->ne[0] % 8 == 0 && (uintptr_t) src->data % 16 == 0 &&
++                        src->nb[1] % 16 == 0 && src->nb[2] % 16 == 0 && src->nb[3] % 16 == 0;
++    if (vec_ok) {
++        const int64_t n8 = ggml_nelements(src) / 8;
++        if (bf16) {
++            cudnn_attn_narrow_f32<nv_bfloat16><<<cudnn_attn_grid(n8), 256, 0, stream>>>(
++                (const char *) src->data, (nv_bfloat16 *) dst, n8, src->ne[0] / 8, src->ne[1], src->ne[2], src->nb[1],
++                src->nb[2], src->nb[3]);
++        } else {
++            cudnn_attn_narrow_f32<half><<<cudnn_attn_grid(n8), 256, 0, stream>>>(
++                (const char *) src->data, (half *) dst, n8, src->ne[0] / 8, src->ne[1], src->ne[2], src->nb[1],
++                src->nb[2], src->nb[3]);
++        }
++        return;
++    }
++    if (bf16) {
++        to_bf16_nc_cuda_t f = ggml_get_to_bf16_nc_cuda(src->type);
++        GGML_ASSERT(f != nullptr);
++        f(src->data, (nv_bfloat16 *) dst, src->ne[0], src->ne[1], src->ne[2], src->ne[3], src->nb[1] / ts,
++          src->nb[2] / ts, src->nb[3] / ts, stream);
++    } else {
++        to_fp16_nc_cuda_t f = ggml_get_to_fp16_nc_cuda(src->type);
++        GGML_ASSERT(f != nullptr);
++        f(src->data, (half *) dst, src->ne[0], src->ne[1], src->ne[2], src->ne[3], src->nb[1] / ts, src->nb[2] / ts,
++          src->nb[3] / ts, stream);
++    }
++}
++
++bool ggml_cuda_cudnn_attn(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
++    static const bool log_calls = cudnn_attn_env_set("GGML_CUDA_CUDNN_ATTN_LOG", false);
++    static std::atomic<long long> n_calls{ 0 };
++
++    cudnn_attn_plan plan;
++    if (!cudnn_attn_make_plan(ctx.device, dst, plan)) {
++        return false;
++    }
++    cudaStream_t stream = ctx.stream();
++
++    // Plans are built outside stream capture only; a captured graph keeps the ggml kernels.
++    cudaStreamCaptureStatus capture = cudaStreamCaptureStatusNone;
++    CUDA_CHECK(cudaStreamIsCapturing(stream, &capture));
++    if (capture != cudaStreamCaptureStatusNone) {
++        return false;
++    }
++
++    const ggml_tensor * Q = dst->src[0];
++    const ggml_tensor * K = dst->src[1];
++    const ggml_tensor * V = dst->src[2];
++    if ((uintptr_t) dst->data % 16 != 0 || (!plan.convert_k && (uintptr_t) K->data % 16 != 0) ||
++        (!plan.convert_v && (uintptr_t) V->data % 16 != 0)) {
++        return false;
++    }
++
++    size_t workspace_size = 0;
++    if (!ggml_cuda_cudnn_sdpa_prepare(plan.p, &workspace_size)) {
++        if (log_calls) {
++            GGML_LOG_INFO("%s: %s %s -> ggml kernel (no cuDNN plan)\n", __func__, ggml_op_desc(dst), dst->name);
++        }
++        return false;
++    }
++
++    char * base    = (char *) dst->data;
++    char * scratch = (char *) GGML_PAD((uintptr_t) dst->data + ggml_nbytes(dst), 256);
++    auto   where   = [&](const cudnn_attn_slot & slot) -> void * {
++        return (slot.in_dst ? base : scratch) + slot.off;
++    };
++    void * q = where(plan.q);
++    void * k = plan.convert_k ? where(plan.k) : K->data;
++    void * v = plan.convert_v ? where(plan.v) : V->data;
++    void * o = where(plan.o);
++
++    cudnn_attn_convert(Q, q, plan.p.bf16, stream);
++    if (plan.convert_k) {
++        cudnn_attn_convert(K, k, plan.p.bf16, stream);
++    }
++    if (plan.convert_v) {
++        cudnn_attn_convert(V, v, plan.p.bf16, stream);
++    }
++
++    ggml_cuda_pool_alloc<char> workspace(ctx.pool());
++    if (workspace_size > 0) {
++        workspace.alloc(workspace_size);
++    }
++    if (!ggml_cuda_cudnn_sdpa_execute(plan.p, stream, q, k, v, o, workspace_size > 0 ? workspace.get() : nullptr)) {
++        return false;
++    }
++
++    // dst holds d * hq * sq * b values with d % 8 == 0, and both buffers are 256-byte aligned.
++    const int64_t n8 = ggml_nelements(dst) / 8;
++    if (plan.p.bf16) {
++        cudnn_attn_widen<<<cudnn_attn_grid(n8), 256, 0, stream>>>((const nv_bfloat16 *) o, (float *) dst->data, n8);
++    } else {
++        cudnn_attn_widen<<<cudnn_attn_grid(n8), 256, 0, stream>>>((const half *) o, (float *) dst->data, n8);
++    }
++    CUDA_CHECK(cudaGetLastError());
++
++    const long long n = ++n_calls;
++    if (log_calls) {
++        GGML_LOG_INFO("%s: %s %s -> cuDNN (call %lld)\n", __func__, ggml_op_desc(dst), dst->name, n);
++    }
++    return true;
++}
++
++#else
++
++size_t ggml_cuda_cudnn_attn_alloc_size(int, const ggml_tensor *) {
++    return 0;
++}
++
++bool ggml_cuda_cudnn_attn(ggml_backend_cuda_context &, ggml_tensor *) {
++    return false;
++}
++
++#endif
+diff --git a/src/ggml-cuda/cudnn-attn.cuh b/src/ggml-cuda/cudnn-attn.cuh
+new file mode 100644
+index 00000000..067bea63
+--- /dev/null
++++ b/src/ggml-cuda/cudnn-attn.cuh
+@@ -0,0 +1,26 @@
++#pragma once
++
++#include "common.cuh"
++
++// cuDNN scaled dot product attention for GGML_OP_FLASH_ATTN_EXT and GGML_OP_SAGE_ATTN on NVIDIA
++// sm80+ when the backend is built with GGML_CUDA_CUDNN and libcudnn 9 loads at run time.
++// Only unmasked, sink-free, ALiBi-free, softcap-free attention with equal Q/K/V head sizes and at
++// least 2^20 attention scores is taken; everything else, and every shape cuDNN rejects, keeps the
++// existing kernels.
++//
++// By default FLASH_ATTN_EXT takes it on sm89 and sm100 and SAGE_ATTN on sm100, where it measured faster.
++// Environment:
++//   GGML_CUDA_CUDNN_ATTN=0|1     never use cuDNN for attention / use it on every sm80+ GPU
++//   GGML_CUDA_CUDNN_SAGE=0|1     keep the SageAttention kernels for GGML_OP_SAGE_ATTN / use cuDNN on every
++//                                sm80+ GPU (AUTO mode; an explicitly requested Sage variant keeps its kernel)
++//   GGML_CUDA_CUDNN_ATTN_BF16=1  run cuDNN in BF16 instead of F16 (F16 K/V are narrowed)
++//   GGML_CUDA_CUDNN_LIB=path     library to open instead of libcudnn.so.9
++//   GGML_CUDA_CUDNN_ATTN_LOG=1   log every call that takes (or falls back from) the cuDNN path
++
++// Bytes the op needs from the graph allocation when the cuDNN path can run, 0 otherwise. The
++// region starts at dst->data and includes dst itself.
++size_t ggml_cuda_cudnn_attn_alloc_size(int device, const ggml_tensor * dst);
++
++// Runs dst through cuDNN. Returns false without touching dst when the cuDNN path does not apply,
++// so the caller runs its own kernel.
++bool ggml_cuda_cudnn_attn(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
+diff --git a/src/ggml-cuda/fattn.cu b/src/ggml-cuda/fattn.cu
+index 9fb380e8..84c4e653 100644
+--- a/src/ggml-cuda/fattn.cu
++++ b/src/ggml-cuda/fattn.cu
+@@ -4,6 +4,7 @@
+ #include "fattn-tile.cuh"
+ #include "fattn-vec.cuh"
+ #include "fattn.cuh"
++#include "cudnn-attn.cuh"
+ 
+ #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+ // one list per group of ncols1 queries: a column is selected if any query of the group can see it
+@@ -760,11 +761,15 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d
+     const ggml_cuda_flash_attn_ext_f16_extra_data f16_extra =
+         ggml_cuda_flash_attn_ext_get_f16_extra_data(dst, need_f16_K, need_f16_V);
+ 
+-    return f16_extra.end - (uintptr_t) dst->data;
++    // The cuDNN path, when it can run, uses the same region after dst for its own scratch.
++    return std::max<size_t>(f16_extra.end - (uintptr_t) dst->data, ggml_cuda_cudnn_attn_alloc_size(device, dst));
+ }
+ 
+ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
+     ggml_cuda_set_device(ctx.device);
++    if (ggml_cuda_cudnn_attn(ctx, dst)) {
++        return;
++    }
+     switch (ggml_cuda_get_best_fattn_kernel(ggml_cuda_get_device(), dst)) {
+         case BEST_FATTN_KERNEL_NONE:
+             GGML_ABORT("fatal error");
+diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu
+index 5c799ccd..f19aba44 100644
+--- a/src/ggml-cuda/ggml-cuda.cu
++++ b/src/ggml-cuda/ggml-cuda.cu
+@@ -961,7 +961,7 @@ static size_t ggml_backend_cuda_buffer_type_get_alloc_size(ggml_backend_buffer_t
+         ? ggml_cuda_flash_attn_ext_get_alloc_size(buft_ctx->device, tensor)
+         : ggml_nbytes(tensor);
+     if (tensor->op == GGML_OP_SAGE_ATTN) {
+-        size = ggml_cuda_sage_attn_get_alloc_size(tensor);
++        size = ggml_cuda_sage_attn_get_alloc_size(buft_ctx->device, tensor);
+     }
+     if (tensor->op == GGML_OP_SOL_ATTN) {
+         size = ggml_cuda_sol_attn_get_alloc_size(tensor);
+diff --git a/src/ggml-cuda/sage-attn.cu b/src/ggml-cuda/sage-attn.cu
+index 2c103eb0..ceb29ed2 100644
+--- a/src/ggml-cuda/sage-attn.cu
++++ b/src/ggml-cuda/sage-attn.cu
+@@ -1,4 +1,5 @@
+ #include "sage-attn.cuh"
++#include "cudnn-attn.cuh"
+ 
+ #if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDACC_VER_MAJOR__ >= 12
+ 
+@@ -332,13 +333,18 @@ bool ggml_cuda_sage_attn_supported(int device, const ggml_tensor* op) {
+            op->ne[2] == q->ne[1] && op->ne[3] == q->ne[3];
+ }
+ 
+-size_t ggml_cuda_sage_attn_get_alloc_size(const ggml_tensor* op) {
++size_t ggml_cuda_sage_attn_get_alloc_size(int device, const ggml_tensor* op) {
+     // Scratch belongs to the graph allocation, so graph capture and VRAM planning see its full lifetime.
+-    return SageWorkspace(op).size;
++    // The cuDNN path, when it can run, reuses the same region for its own scratch.
++    return std::max<size_t>(SageWorkspace(op).size, ggml_cuda_cudnn_attn_alloc_size(device, op));
+ }
+ 
+ void ggml_cuda_sage_attn(ggml_backend_cuda_context& ctx, ggml_tensor* dst) {
+     GGML_ASSERT(ggml_cuda_sage_attn_supported(ctx.device, dst));
++    ggml_cuda_set_device(ctx.device);
++    if (ggml_cuda_cudnn_attn(ctx, dst)) {
++        return;
++    }
+     if (dst->src[0]->ne[0] == 64) {
+         sage_attn<64>(ctx, dst);
+     } else {
+@@ -352,7 +358,7 @@ bool ggml_cuda_sage_attn_supported(int, const ggml_tensor*) {
+     return false;
+ }
+ 
+-size_t ggml_cuda_sage_attn_get_alloc_size(const ggml_tensor* op) {
++size_t ggml_cuda_sage_attn_get_alloc_size(int, const ggml_tensor* op) {
+     return ggml_nbytes(op);
+ }
+ 
+diff --git a/src/ggml-cuda/sage-attn.cuh b/src/ggml-cuda/sage-attn.cuh
+index 126c7a74..e2648857 100644
+--- a/src/ggml-cuda/sage-attn.cuh
++++ b/src/ggml-cuda/sage-attn.cuh
+@@ -4,7 +4,7 @@
+ #include "common.cuh"
+ 
+ bool ggml_cuda_sage_attn_supported(int device, const ggml_tensor* op);
+-size_t ggml_cuda_sage_attn_get_alloc_size(const ggml_tensor* op);
++size_t ggml_cuda_sage_attn_get_alloc_size(int device, const ggml_tensor* op);
+ void ggml_cuda_sage_attn(ggml_backend_cuda_context& ctx, ggml_tensor* dst);
+ 
+ #endif
+diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
+index 108da4a2..62be5695 100644
+--- a/tests/test-backend-ops.cpp
++++ b/tests/test-backend-ops.cpp
+@@ -12008,6 +12008,21 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
+     test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256,  4, {4, 1}, 1024, 75, true, false, 0, 10.0f, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
+     test_cases.emplace_back(new test_flash_attn_ext_large_logits(256, 256,  4, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16));
+ 
++    // unmasked F16 K/V with the MiniMax-H3 DiT head layout (42 heads x 128), the shape class the CUDA
++    // cuDNN attention path takes: odd and tile-aligned lengths, GQA with batch, F32 K/V, head size 64.
++    for (int64_t n : { 1023, 4096 }) {
++        test_cases.emplace_back(new test_flash_attn_ext(128, 128, 42, {1, 1}, n, n, false, false, 0, 0,
++                                                        GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
++    }
++    test_cases.emplace_back(new test_flash_attn_ext(128, 128, 42, {1, 1}, 4096, 333, false, false, 0, 0,
++                                                    GGML_PREC_DEFAULT, GGML_TYPE_F16, GGML_TYPE_F16));
++    test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {4, 2}, 777, 333, false, false, 0, 0,
++                                                    GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
++    test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 512, 512, false, false, 0, 0,
++                                                    GGML_PREC_F32, GGML_TYPE_F32, GGML_TYPE_F32));
++    test_cases.emplace_back(new test_flash_attn_ext(128, 128, 42, {1, 1}, 1023, 1023, false, false, 0, 0,
++                                                    GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16, {0, 2, 1, 3}));
++
+     test_cases.emplace_back(new test_cross_entropy_loss     (GGML_TYPE_F32, {   10, 5, 4, 3}));
+     test_cases.emplace_back(new test_cross_entropy_loss     (GGML_TYPE_F32, {30000, 1, 1, 1}));
+     test_cases.emplace_back(new test_cross_entropy_loss_back(GGML_TYPE_F32, {   10, 5, 4, 3}));
+@@ -12469,6 +12484,16 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
+         test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {12, 1}, kv, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0, {0, 1, 2, 3}, true, false, 0));
+     }
+ 
++    // unmasked diffusion attention: MiniMax-H3 DiT (19315 tokens x 56 heads at 960x544x124; 22440 x 42),
++    // its token refiner (31 tokens) and video VAE decoder (1797 x 32 x 64), FLUX-class 1024px (4096 x 24),
++    // and a single-query row
++    for (auto [hs, nh, n] : std::vector<std::array<int64_t, 3>>{ { 128, 56, 19315 }, { 128, 42, 22440 }, { 128, 56, 31 },
++                                                                 { 64, 32, 1797 }, { 128, 24, 4096 }, { 128, 16, 256 },
++                                                                 { 128, 16, 512 }, { 128, 16, 1024 } }) {
++        test_cases.emplace_back(new test_flash_attn_ext(hs, hs, nh, {1, 1}, n, n, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
++    }
++    test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {4, 1}, 4096, 1, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
++
+     test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
+     test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
+     test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680,   1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0));
diff --git a/scripts/unsloth/ggml-patches/0007-ggml-cuda-generic-dit-vae.patch b/scripts/unsloth/ggml-patches/0007-ggml-cuda-generic-dit-vae.patch
new file mode 100644
index 0000000000..79a61e5495
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0007-ggml-cuda-generic-dit-vae.patch
@@ -0,0 +1,1133 @@
+diff --git a/include/ggml.h b/include/ggml.h
+index f7ef1573..bec0b7b6 100644
+--- a/include/ggml.h
++++ b/include/ggml.h
+@@ -2581,6 +2581,14 @@ extern "C" {
+     // ggml_rope_pe_permute of ggml_rms_norm(a, eps) (no weight): every head row of `a` is
+     // RMS-normalised first, with the same arithmetic as GGML_OP_RMS_NORM on the same backend, so
+     // the result matches the two-op chain bit for bit where the backend supports it.
++    // ggml_rope_pe_permute (or _rms) with interleaved pairing: rotated element i < n_rot pairs
++    // a[2*j] with a[2*j + 1] (j = i / 2, r = i % 2): y = a[2*j]*pe[0, r, j, t] + a[2*j + 1]*pe[1, r, j, t],
++    // the pairing ggml_rope_apply(interleaved = true) uses, with the same rounding (bit-identical
++    // to ggml_rope_apply -> ggml_scale -> ggml_cpy). Returns `t` (a ggml_rope_pe_permute result).
++    GGML_API struct ggml_tensor * ggml_rope_pe_permute_set_interleaved(
++            struct ggml_tensor  * t,
++            bool                  interleaved);
++
+     GGML_API struct ggml_tensor * ggml_rope_pe_permute_rms(
+             struct ggml_context * ctx,
+             struct ggml_tensor  * a,
+diff --git a/src/ggml-cpu/ops.cpp b/src/ggml-cpu/ops.cpp
+index ebe6e202..f7d6b8f7 100644
+--- a/src/ggml-cpu/ops.cpp
++++ b/src/ggml-cpu/ops.cpp
+@@ -12703,6 +12703,7 @@ void ggml_compute_forward_rope_pe_permute(
+     const float scale = ggml_get_op_params_f32(dst, 1);
+     const bool  norm  = ggml_get_op_params_i32(dst, 2) != 0;
+     const float eps   = ggml_get_op_params_f32(dst, 3);
++    const bool  inter = ggml_get_op_params_i32(dst, 4) != 0;
+     const int   half  = n_rot / 2;
+ 
+     const int64_t D = src0->ne[0];
+@@ -12741,17 +12742,28 @@ void ggml_compute_forward_rope_pe_permute(
+ 
+         if (n_rot > 0) {
+             const float * t = (const float *) pe->data + l * (int64_t) half * 4;
+-            for (int i = 0; i < n_rot; ++i) {
+-                const int r = i / half;
+-                const int j = i - r * half;
+-                p0[i]       = x[j] * t[4 * j + 2 * r + 0];
+-            }
+-            for (int i = 0; i < n_rot; ++i) {
+-                const int r = i / half;
+-                const int j = i - r * half;
+-                p1[i]       = x[j + half] * t[4 * j + 2 * r + 1];
++            if (inter) {
++                // the expression ggml_compute_forward_rope_apply (interleaved) evaluates, so the compiler
++                // contracts it (or not) the same way and the two ops stay bit-identical on this backend
++                for (int j = 0; j < half; ++j) {
++                    const float a    = x[2 * j];
++                    const float b    = x[2 * j + 1];
++                    y[2 * j]         = a*t[4 * j + 0] + b*t[4 * j + 1];
++                    y[2 * j + 1]     = a*t[4 * j + 2] + b*t[4 * j + 3];
++                }
++            } else {
++                for (int i = 0; i < n_rot; ++i) {
++                    const int r = i / half;
++                    const int j = i - r * half;
++                    p0[i]       = x[j] * t[4 * j + 2 * r + 0];
++                }
++                for (int i = 0; i < n_rot; ++i) {
++                    const int r = i / half;
++                    const int j = i - r * half;
++                    p1[i]       = x[j + half] * t[4 * j + 2 * r + 1];
++                }
++                ggml_vec_add_f32(n_rot, y.data(), p0.data(), p1.data());
+             }
+-            ggml_vec_add_f32(n_rot, y.data(), p0.data(), p1.data());
+         }
+         if (D > n_rot) {
+             memcpy(y.data() + n_rot, x + n_rot, (D - n_rot) * sizeof(float));
+diff --git a/src/ggml-cuda/ggml-cuda.cu b/src/ggml-cuda/ggml-cuda.cu
+index f19aba44..05ac1936 100644
+--- a/src/ggml-cuda/ggml-cuda.cu
++++ b/src/ggml-cuda/ggml-cuda.cu
+@@ -37,6 +37,9 @@
+ #include "ggml-cuda/mmvq.cuh"
+ #include "ggml-cuda/moe-weighted-reduction.cuh"
+ #include "ggml-cuda/norm.cuh"
++#include "ggml-cuda/group-norm-par.cuh"
++#include "ggml-cuda/norm-channel.cuh"
++#include "ggml-cuda/mul-add-fused.cuh"
+ #include "ggml-cuda/opt-step-adamw.cuh"
+ #include "ggml-cuda/opt-step-sgd.cuh"
+ #include "ggml-cuda/out-prod.cuh"
+@@ -4538,6 +4541,106 @@ static int ggml_cuda_try_fuse_cublas_epilogue(ggml_backend_cuda_context & ctx, g
+     return consumed;
+ }
+ 
++// GROUP_NORM -> MUL(w) -> ADD(b) [-> SILU] with w, b per channel ([1, 1, C, 1], F32, contiguous): the
++// shape ggml_ext_group_norm-style VAE / U-Net code builds. Returns the number of extra nodes consumed.
++static int ggml_cuda_try_fuse_group_norm(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i) {
++    ggml_tensor * gn = cgraph->nodes[i];
++    if (gn->op != GGML_OP_GROUP_NORM || !ggml_cuda_group_norm_par_supported(gn)) {
++        return 0;
++    }
++    // the per-channel operands are usually reshaped 1-D weights; their RESHAPE nodes sit between the
++    // ops of the chain, so step over view nodes that do not read the chain itself
++    auto next_op = [&](int from, const ggml_tensor * prev) {
++        for (int j = from + 1; j < cgraph->n_nodes; ++j) {
++            const ggml_tensor * n = cgraph->nodes[j];
++            const bool view = n->op == GGML_OP_RESHAPE || n->op == GGML_OP_VIEW || n->op == GGML_OP_PERMUTE ||
++                              n->op == GGML_OP_TRANSPOSE;
++            if (view && n->src[0] != prev && n->view_src != prev) {
++                continue;
++            }
++            return j;
++        }
++        return -1;
++    };
++    const int i_mul = next_op(i, gn);
++    const int i_add = i_mul < 0 ? -1 : next_op(i_mul, cgraph->nodes[i_mul]);
++    if (i_add < 0) {
++        return 0;
++    }
++    int idx[4] = { i, i_mul, i_add, -1 };
++    const enum ggml_op ops[4] = { GGML_OP_GROUP_NORM, GGML_OP_MUL, GGML_OP_ADD, GGML_OP_UNARY };
++    // ggml_can_fuse_ext minus its view rule: sd.cpp's in-place MUL/ADD/SILU are views of the GROUP_NORM
++    // result, every reader of that memory is a counted chain use, and the fused kernel writes the same values.
++    auto chain_ok = [&](int n) {
++        for (int k = 0; k < n; ++k) {
++            const ggml_tensor * node = cgraph->nodes[idx[k]];
++            if (node->op != ops[k] || (node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
++                return false;
++            }
++            if (k == 0 ? node->view_src != nullptr
++                       : node->view_src != nullptr && (node->view_src != gn || node->view_offs != 0 || node->data != gn->data)) {
++                return false;
++            }
++            if (k < n - 1 && (ggml_node_get_use_count(cgraph, idx[k]) != 1 || (node->flags & GGML_TENSOR_FLAG_OUTPUT))) {
++                return false;
++            }
++            if (k > 0) {
++                const ggml_tensor * prev = cgraph->nodes[idx[k - 1]];
++                if ((node->src[0] != prev && node->src[1] != prev) || !ggml_are_same_shape(node, prev)) {
++                    return false;
++                }
++            }
++        }
++        return true;
++    };
++    if (!chain_ok(3)) {
++        return 0;
++    }
++    ggml_tensor * mul = cgraph->nodes[i_mul];
++    ggml_tensor * add = cgraph->nodes[i_add];
++    const ggml_tensor * w = mul->src[0] == gn ? mul->src[1] : mul->src[0];
++    const ggml_tensor * b = add->src[0] == mul ? add->src[1] : add->src[0];
++    const int64_t C = gn->ne[2];
++    auto per_channel = [&](const ggml_tensor * t) {
++        return t != nullptr && t->type == GGML_TYPE_F32 && ggml_is_contiguous(t) && t->ne[0] == 1 && t->ne[1] == 1 &&
++               t->ne[2] == C && t->ne[3] == 1;
++    };
++    if (!per_channel(w) || !per_channel(b) || mul->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 ||
++        !ggml_is_contiguous(mul) || !ggml_is_contiguous(add)) {
++        return 0;
++    }
++    int n_fused = 3;
++    const int i_u = next_op(i_add, add);
++    if (i_u > 0) {
++        const ggml_tensor * u = cgraph->nodes[i_u];
++        idx[3] = i_u;
++        if (u->op == GGML_OP_UNARY && ggml_get_unary_op(u) == GGML_UNARY_OP_SILU && u->type == GGML_TYPE_F32 &&
++            ggml_is_contiguous(u) && chain_ok(4)) {
++            n_fused = 4;
++        }
++    }
++    ggml_tensor * out = cgraph->nodes[idx[n_fused - 1]];
++    // every node in [i, last] is either fused here or a view the loop would skip anyway
++    const int last = idx[n_fused - 1];
++    int out_nodes[] = { last };
++    if (!ggml_cuda_check_fusion_memory_ranges(cgraph, i, last - i + 1, out_nodes, 1)) {
++        return 0;
++    }
++    // leaf sources skipped above: w/b must not overlap out; x only elementwise (x[j] read before out[j] written)
++    auto overlaps = [&](const ggml_tensor * t) {
++        const char * o0 = (const char *) out->data;
++        const char * t0 = (const char *) t->data;
++        return t0 < o0 + ggml_nbytes(out) && o0 < t0 + ggml_nbytes(t);
++    };
++    const ggml_tensor * x = gn->src[0];
++    if (overlaps(w) || overlaps(b) ||
++        (overlaps(x) && !(x->data == out->data && ggml_are_same_shape(x, out) && ggml_is_contiguous(x)))) {
++        return 0;
++    }
++    ggml_cuda_op_group_norm_par(ctx, gn, w, b, n_fused == 4, out);
++    return last - i;
++}
++
+ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) {
+ 
+     if (ggml_cuda_fusion_disabled()) {
+@@ -4566,6 +4669,20 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
+         }
+     }
+ 
++    if (node->op == GGML_OP_GROUP_NORM) {
++        const int consumed = ggml_cuda_try_fuse_group_norm(*cuda_ctx, cgraph, i);
++        if (consumed > 0) {
++            return consumed;
++        }
++    }
++
++    if (node->op == GGML_OP_CONT) {
++        const int consumed = ggml_cuda_try_fuse_rms_norm_channel(*cuda_ctx, cgraph, i);
++        if (consumed > 0) {
++            return consumed;
++        }
++    }
++
+     if (node->op == GGML_OP_MUL) {
+         ggml_cuda_moe_weighted_reduction_match match;
+         if (ggml_cuda_match_moe_weighted_reduction(cgraph, i, match)) {
+@@ -4578,6 +4695,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
+         }
+     }
+ 
++    if (node->op == GGML_OP_MUL) {
++        const int consumed = ggml_cuda_try_fuse_mul_add(*cuda_ctx, cgraph, i);
++        if (consumed > 0) {
++            return consumed;
++        }
++    }
++
+     // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache
+     if (node->op == GGML_OP_GATED_DELTA_NET) {
+         ggml_cuda_gated_delta_net_fused_cache fused_state_cpy;
+diff --git a/src/ggml-cuda/group-norm-par.cu b/src/ggml-cuda/group-norm-par.cu
+new file mode 100644
+index 00000000..dbc652df
+--- /dev/null
++++ b/src/ggml-cuda/group-norm-par.cu
+@@ -0,0 +1,174 @@
++#include "group-norm-par.cuh"
++#include "unary.cuh"
++
++// The stock group_norm_f32<1024> runs one 1024-thread block per group, so a VAE decode with 32
++// groups keeps 32 SMs busy and walks the group three times (sum, centred sum of squares, scale),
++// then the affine and the SiLU are three more full passes. Here the same reductions run spread over
++// the GPU without changing a single rounding:
++//   - thread t (0..1023) of the stock block sums x[g*G + t + 1024*k] in increasing k; each such
++//     chain is computed whole by one thread here too, 32 consecutive t per warp, so the warp sums
++//     (xor butterfly) are the same values the stock kernel writes to shared memory;
++//   - the 32 warp sums are combined by the same warp butterfly (stock: warp lanes read shared[lane]);
++//   - mean = sum / G and the centred pass use the same expressions, then scale = rsqrtf(var + eps);
++//   - y = (x - mean) * scale, then * w, + b, then silu, each rounded separately as in the
++//     unfused MUL / ADD / UNARY kernels.
++// Kernels: partial sums, partial centred squares, per-group stats, one elementwise apply pass.
++
++static constexpr int GN_PAR_THREADS = 1024; // the stock block size whose arithmetic is reproduced
++static constexpr int GN_PAR_WARPS   = GN_PAR_THREADS / WARP_SIZE;
++static constexpr int GN_PAR_WPB     = 4;    // warps per block in the reduction kernels
++
++bool ggml_cuda_group_norm_par_enabled() {
++#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
++    return false;  // warp size / reduction differ from what is reproduced here
++#else
++    static const bool enabled = [] {
++        const char * e = getenv("GGML_CUDA_GROUP_NORM_PAR");
++        return e == nullptr || atoi(e) != 0;
++    }();
++    return enabled && WARP_SIZE == 32;
++#endif
++}
++
++static int64_t gn_par_group_size(const ggml_tensor * gn) {
++    const ggml_tensor * src0       = gn->src[0];
++    const int           num_groups = gn->op_params[0];
++    return src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups);
++}
++
++bool ggml_cuda_group_norm_par_supported(const ggml_tensor * gn) {
++    if (!ggml_cuda_group_norm_par_enabled() || gn->op != GGML_OP_GROUP_NORM) {
++        return false;
++    }
++    const ggml_tensor * src0 = gn->src[0];
++    if (src0->type != GGML_TYPE_F32 || gn->type != GGML_TYPE_F32 || !ggml_is_contiguous(src0) || !ggml_is_contiguous(gn)) {
++        return false;
++    }
++    const int64_t group_size = gn_par_group_size(gn);
++    // the stock kernel uses its 1024-thread variant only from 1024 elements per group; it indexes with int
++    return group_size >= GN_PAR_THREADS && ggml_nelements(src0) < INT_MAX;
++}
++
++// one warp per (group, chain block of 32 threads): sum of x over the warp's 32 chains
++static __global__ void gn_par_sum(const float * __restrict__ x, float * __restrict__ part, const int group_size,
++                                  const int ne_elements) {
++    const int g    = blockIdx.x;
++    const int w    = blockIdx.y * GN_PAR_WPB + threadIdx.x / WARP_SIZE;
++    const int t    = w * WARP_SIZE + threadIdx.x % WARP_SIZE;
++    const int start = g * group_size + t;
++    const int end   = min(g * group_size + group_size, ne_elements);
++    float tmp = 0.0f;
++    for (int j = start; j < end; j += GN_PAR_THREADS) {
++        tmp += x[j];
++    }
++    tmp = warp_reduce_sum(tmp);
++    if (threadIdx.x % WARP_SIZE == 0) {
++        part[g * GN_PAR_WARPS + w] = tmp;
++    }
++}
++
++static __device__ __forceinline__ float gn_par_combine(const float * part, const int g) {
++    // stock: lanes read shared_vals[lane] (one value per warp) and reduce
++    return warp_reduce_sum(part[g * GN_PAR_WARPS + threadIdx.x % WARP_SIZE]);
++}
++
++static __global__ void gn_par_sqdev(const float * __restrict__ x, const float * __restrict__ part, float * __restrict__ part2,
++                                    const int group_size, const int ne_elements) {
++    const int g    = blockIdx.x;
++    const int w    = blockIdx.y * GN_PAR_WPB + threadIdx.x / WARP_SIZE;
++    const int t    = w * WARP_SIZE + threadIdx.x % WARP_SIZE;
++    const int start = g * group_size + t;
++    const int end   = min(g * group_size + group_size, ne_elements);
++
++    float tmp = gn_par_combine(part, g);
++    const float mean = tmp / group_size;
++    tmp = 0.0f;
++    for (int j = start; j < end; j += GN_PAR_THREADS) {
++        const float xi = x[j] - mean;
++        tmp += xi * xi;
++    }
++    tmp = warp_reduce_sum(tmp);
++    if (threadIdx.x % WARP_SIZE == 0) {
++        part2[g * GN_PAR_WARPS + w] = tmp;
++    }
++}
++
++static __global__ void gn_par_stats(const float * __restrict__ part, const float * __restrict__ part2, float2 * __restrict__ stats,
++                                    const int group_size, const float eps) {
++    const int g = blockIdx.x;
++    float tmp = gn_par_combine(part, g);
++    const float mean = tmp / group_size;
++    tmp = gn_par_combine(part2, g);
++    const float variance = tmp / group_size;
++    const float scale    = rsqrtf(variance + eps);
++    if (threadIdx.x == 0) {
++        stats[g] = make_float2(mean, scale);
++    }
++}
++
++template <bool affine, bool silu>
++static __global__ void gn_par_apply(const float * x, const float2 * __restrict__ stats, const float * __restrict__ w,
++                                    const float * __restrict__ b, float * dst, const int64_t ne_elements,
++                                    const int group_size, const int64_t plane, const int64_t channels) {
++    const int64_t stride = (int64_t) gridDim.x * blockDim.x;
++    for (int64_t j = (int64_t) blockIdx.x * blockDim.x + threadIdx.x; j < ne_elements; j += stride) {
++        const float2 st = stats[j / group_size];
++        float y = __fmul_rn(__fsub_rn(x[j], st.x), st.y);
++        if constexpr (affine) {
++            const int64_t c = (j / plane) % channels;
++            y = __fmul_rn(y, w[c]);
++            y = __fadd_rn(y, b[c]);
++        }
++        if constexpr (silu) {
++            y = ggml_cuda_op_silu_single(y);
++        }
++        dst[j] = y;
++    }
++}
++
++void ggml_cuda_op_group_norm_par(ggml_backend_cuda_context & ctx, const ggml_tensor * gn, const ggml_tensor * w,
++                                 const ggml_tensor * b, bool silu, ggml_tensor * out) {
++    const ggml_tensor * src0 = gn->src[0];
++    const float * x = (const float *) src0->data;
++    float * dst = (float *) out->data;
++    cudaStream_t stream = ctx.stream();
++
++    const int num_groups = gn->op_params[0];
++    float eps;
++    memcpy(&eps, gn->op_params + 1, sizeof(float));
++    GGML_ASSERT(eps >= 0.0f);
++
++    const int group_size  = (int) gn_par_group_size(gn);
++    const int ne_elements = (int) ggml_nelements(src0);
++    const int n_groups    = num_groups * (int) src0->ne[3];
++
++    ggml_cuda_pool_alloc<float>  part(ctx.pool(), (size_t) n_groups * GN_PAR_WARPS);
++    ggml_cuda_pool_alloc<float>  part2(ctx.pool(), (size_t) n_groups * GN_PAR_WARPS);
++    ggml_cuda_pool_alloc<float2> stats(ctx.pool(), (size_t) n_groups);
++
++    const dim3 rgrid(n_groups, GN_PAR_WARPS / GN_PAR_WPB, 1);
++    const dim3 rblock(GN_PAR_WPB * WARP_SIZE, 1, 1);
++    gn_par_sum<<<rgrid, rblock, 0, stream>>>(x, part.get(), group_size, ne_elements);
++    gn_par_sqdev<<<rgrid, rblock, 0, stream>>>(x, part.get(), part2.get(), group_size, ne_elements);
++    gn_par_stats<<<n_groups, WARP_SIZE, 0, stream>>>(part.get(), part2.get(), stats.get(), group_size, eps);
++
++    const int     threads = 256;
++    const int64_t want    = ((int64_t) ne_elements + threads - 1) / threads;
++    const int     blocks  = (int) std::min<int64_t>(want, (int64_t) ggml_cuda_info().devices[ctx.device].nsm * 32);
++    const int64_t plane   = src0->ne[0] * src0->ne[1];
++    const float * wd = w ? (const float *) w->data : nullptr;
++    const float * bd = b ? (const float *) b->data : nullptr;
++    if (w) {
++        if (silu) {
++            gn_par_apply<true, true><<<blocks, threads, 0, stream>>>(x, stats.get(), wd, bd, dst, ne_elements, group_size, plane, src0->ne[2]);
++        } else {
++            gn_par_apply<true, false><<<blocks, threads, 0, stream>>>(x, stats.get(), wd, bd, dst, ne_elements, group_size, plane, src0->ne[2]);
++        }
++    } else {
++        if (silu) {
++            gn_par_apply<false, true><<<blocks, threads, 0, stream>>>(x, stats.get(), wd, bd, dst, ne_elements, group_size, plane, src0->ne[2]);
++        } else {
++            gn_par_apply<false, false><<<blocks, threads, 0, stream>>>(x, stats.get(), wd, bd, dst, ne_elements, group_size, plane, src0->ne[2]);
++        }
++    }
++}
+diff --git a/src/ggml-cuda/group-norm-par.cuh b/src/ggml-cuda/group-norm-par.cuh
+new file mode 100644
+index 00000000..eb6298ea
+--- /dev/null
++++ b/src/ggml-cuda/group-norm-par.cuh
+@@ -0,0 +1,15 @@
++#pragma once
++#include "common.cuh"
++
++// Parallel GROUP_NORM with the stock kernel's exact arithmetic, optionally fused with the
++// per-channel affine (MUL by w, ADD of b, both [1, 1, C, 1]) and a SiLU that follow it in the graph.
++// Returns false (nothing launched) when the inputs are outside what it handles; the caller then
++// runs the stock ops. GGML_CUDA_GROUP_NORM_PAR=0 turns it off.
++bool ggml_cuda_group_norm_par_enabled();
++
++bool ggml_cuda_group_norm_par_supported(const ggml_tensor * gn);
++
++// gn: the GROUP_NORM node; w / b: affine operands or nullptr; silu: apply SiLU; out: the tensor
++// that receives the result (the last fused node).
++void ggml_cuda_op_group_norm_par(ggml_backend_cuda_context & ctx, const ggml_tensor * gn, const ggml_tensor * w,
++                                 const ggml_tensor * b, bool silu, ggml_tensor * out);
+diff --git a/src/ggml-cuda/mul-add-fused.cu b/src/ggml-cuda/mul-add-fused.cu
+new file mode 100644
+index 00000000..29484ce4
+--- /dev/null
++++ b/src/ggml-cuda/mul-add-fused.cu
+@@ -0,0 +1,172 @@
++#include "mul-add-fused.cuh"
++#include "ggml-impl.h"
++
++static bool mul_add_fusion_enabled() {
++    static const bool enabled = [] {
++        const char * e = getenv("GGML_CUDA_FUSE_MUL_ADD");
++        return e == nullptr || atoi(e) != 0;
++    }();
++    return enabled;
++}
++
++struct mad_operand {
++    const float * p;
++    int64_t ne[4];
++    int64_t nb[4]; // in elements
++};
++
++static __device__ __forceinline__ const float * mad_row(const mad_operand & o, const int64_t i1, const int64_t i2, const int64_t i3) {
++    return o.p + (o.ne[1] == 1 ? 0 : i1 * o.nb[1]) + (o.ne[2] == 1 ? 0 : i2 * o.nb[2]) + (o.ne[3] == 1 ? 0 : i3 * o.nb[3]);
++}
++
++static __device__ __forceinline__ float mad_at(const float * row, const mad_operand & o, const int64_t i0) {
++    return row[o.ne[0] == 1 ? 0 : i0 * o.nb[0]];
++}
++
++// one block per output row; y = (a*b) + c [+ d], b, c, d broadcast like binbcast (dims of size 1)
++template <bool has_d>
++static __global__ void mul_add_fused_f32(const mad_operand a, const mad_operand b, const mad_operand c, const mad_operand d,
++                                         float * __restrict__ dst, const int64_t ne0, const int64_t ne1, const int64_t ne2) {
++    const int64_t r  = blockIdx.x;
++    const int64_t i1 = r % ne1;
++    const int64_t i2 = (r / ne1) % ne2;
++    const int64_t i3 = r / (ne1 * ne2);
++    const float * ra = mad_row(a, i1, i2, i3);
++    const float * rb = mad_row(b, i1, i2, i3);
++    const float * rc = mad_row(c, i1, i2, i3);
++    const float * rd = has_d ? mad_row(d, i1, i2, i3) : nullptr;
++    float * out = dst + r * ne0;
++    for (int64_t i0 = threadIdx.x; i0 < ne0; i0 += blockDim.x) {
++        float y = __fmul_rn(mad_at(ra, a, i0), mad_at(rb, b, i0));
++        y = __fadd_rn(y, mad_at(rc, c, i0));
++        if constexpr (has_d) {
++            y = __fadd_rn(y, mad_at(rd, d, i0));
++        }
++        out[i0] = y;
++    }
++}
++
++static bool mad_ok_operand(const ggml_tensor * t, const ggml_tensor * out) {
++    if (t == nullptr || t->type != GGML_TYPE_F32 || t->nb[0] != sizeof(float)) {
++        return false;
++    }
++    for (int k = 0; k < 4; ++k) {
++        if (t->ne[k] != out->ne[k] && t->ne[k] != 1) {
++            return false;
++        }
++        if (t->nb[k] % sizeof(float) != 0) {
++            return false;
++        }
++    }
++    return true;
++}
++
++static mad_operand mad_make(const ggml_tensor * t) {
++    mad_operand o;
++    o.p = (const float *) t->data;
++    for (int k = 0; k < 4; ++k) {
++        o.ne[k] = t->ne[k];
++        o.nb[k] = t->nb[k] / (int64_t) sizeof(float);
++    }
++    return o;
++}
++
++static bool mad_is_skippable_view(const ggml_tensor * n, const ggml_tensor * prev) {
++    const bool view = n->op == GGML_OP_RESHAPE || n->op == GGML_OP_VIEW || n->op == GGML_OP_PERMUTE || n->op == GGML_OP_TRANSPOSE;
++    return view && n->src[0] != prev && n->view_src != prev;
++}
++
++int ggml_cuda_try_fuse_mul_add(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i) {
++    if (!mul_add_fusion_enabled()) {
++        return 0;
++    }
++    ggml_tensor * mul = cgraph->nodes[i];
++    if (mul->op != GGML_OP_MUL || mul->type != GGML_TYPE_F32 || !ggml_is_contiguous(mul) || !ggml_node_has_n_uses(cgraph, i, 1)) {
++        return 0;
++    }
++    auto next_op = [&](int from) {
++        for (int j = from + 1; j < cgraph->n_nodes; ++j) {
++            if (!mad_is_skippable_view(cgraph->nodes[j], cgraph->nodes[from])) {
++                return j;
++            }
++        }
++        return -1;
++    };
++    // the full-shape factor goes first so the output shape is the MUL's shape
++    const ggml_tensor * a = mul->src[0];
++    const ggml_tensor * b = mul->src[1];
++    if (!ggml_are_same_shape(a, mul)) {
++        std::swap(a, b);
++    }
++    if (!ggml_are_same_shape(a, mul) || !mad_ok_operand(a, mul) || !mad_ok_operand(b, mul)) {
++        return 0;
++    }
++    const int i_add = next_op(i);
++    if (i_add < 0) {
++        return 0;
++    }
++    ggml_tensor * add = cgraph->nodes[i_add];
++    // under ggml_build_forward_select only GGML_TENSOR_FLAG_COMPUTE nodes run: never fuse an inactive node
++    if (add->op != GGML_OP_ADD || (add->flags & GGML_TENSOR_FLAG_COMPUTE) == 0 || add->type != GGML_TYPE_F32 ||
++        !ggml_is_contiguous(add) || !ggml_are_same_shape(add, mul) || (add->src[0] != mul && add->src[1] != mul)) {
++        return 0;
++    }
++    const ggml_tensor * c = add->src[0] == mul ? add->src[1] : add->src[0];
++    if (c == mul || !mad_ok_operand(c, add)) {
++        return 0;
++    }
++    int last = i_add;
++    const ggml_tensor * d = nullptr;
++    if (ggml_node_has_n_uses(cgraph, i_add, 1)) {
++        const int i_add2 = next_op(i_add);
++        if (i_add2 > 0) {
++            ggml_tensor * add2 = cgraph->nodes[i_add2];
++            if (add2->op == GGML_OP_ADD && (add2->flags & GGML_TENSOR_FLAG_COMPUTE) && add2->type == GGML_TYPE_F32 && ggml_is_contiguous(add2) && ggml_are_same_shape(add2, add) &&
++                (add2->src[0] == add || add2->src[1] == add)) {
++                const ggml_tensor * dd = add2->src[0] == add ? add2->src[1] : add2->src[0];
++                if (dd != add && dd != mul && mad_ok_operand(dd, add2)) {
++                    d    = dd;
++                    last = i_add2;
++                }
++            }
++        }
++    }
++    for (int j = i + 1; j < last; ++j) {
++        const ggml_tensor * n = cgraph->nodes[j];
++        if (j != i_add && !(n->op == GGML_OP_RESHAPE || n->op == GGML_OP_VIEW || n->op == GGML_OP_PERMUTE || n->op == GGML_OP_TRANSPOSE)) {
++            return 0;
++        }
++    }
++    ggml_tensor * out = cgraph->nodes[last];
++    // the output may only alias the elementwise inputs at the same element (in-place adds)
++    auto overlaps = [&](const ggml_tensor * t) {
++        if (t == nullptr || t->data == nullptr) {
++            return false;
++        }
++        const char * o0 = (const char *) out->data;
++        const char * o1 = o0 + ggml_nbytes(out);
++        const char * t0 = (const char *) t->data;
++        const char * t1 = t0 + ggml_nbytes(t);
++        return t0 < o1 && o0 < t1;
++    };
++    auto same_layout = [&](const ggml_tensor * t) {
++        return t->data == out->data && ggml_are_same_shape(t, out) && ggml_is_contiguous(t);
++    };
++    for (const ggml_tensor * t : { a, b, c, d }) {
++        if (overlaps(t) && !same_layout(t)) {
++            return 0;
++        }
++    }
++    const int64_t nrows = ggml_nrows(out);
++    if (nrows > INT_MAX) {
++        return 0;
++    }
++    const int threads = (int) std::min<int64_t>(256, std::max<int64_t>(32, (out->ne[0] + 31) / 32 * 32));
++    const mad_operand oa = mad_make(a), ob = mad_make(b), oc = mad_make(c), od = d ? mad_make(d) : oc;
++    if (d) {
++        mul_add_fused_f32<true><<<(unsigned int) nrows, threads, 0, ctx.stream()>>>(oa, ob, oc, od, (float *) out->data, out->ne[0], out->ne[1], out->ne[2]);
++    } else {
++        mul_add_fused_f32<false><<<(unsigned int) nrows, threads, 0, ctx.stream()>>>(oa, ob, oc, od, (float *) out->data, out->ne[0], out->ne[1], out->ne[2]);
++    }
++    return last - i;
++}
+diff --git a/src/ggml-cuda/mul-add-fused.cuh b/src/ggml-cuda/mul-add-fused.cuh
+new file mode 100644
+index 00000000..8d9752ca
+--- /dev/null
++++ b/src/ggml-cuda/mul-add-fused.cuh
+@@ -0,0 +1,8 @@
++#pragma once
++#include "common.cuh"
++
++// MUL -> ADD [-> ADD] chains of F32 elementwise ops with broadcast operands (adaLN modulate
++// x + x*scale (+ shift), gated residual r + y*gate): one pass instead of two or three, each
++// product and sum still rounded separately (bit-identical to the unfused MUL / ADD kernels).
++// Returns the number of extra nodes consumed. GGML_CUDA_FUSE_MUL_ADD=0 turns it off.
++int ggml_cuda_try_fuse_mul_add(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i);
+diff --git a/src/ggml-cuda/norm-channel.cu b/src/ggml-cuda/norm-channel.cu
+new file mode 100644
+index 00000000..c1a8819d
+--- /dev/null
++++ b/src/ggml-cuda/norm-channel.cu
+@@ -0,0 +1,259 @@
++#include "norm-channel.cuh"
++#include "ggml-impl.h"
++
++// The unfused RMS_NORM over rows of C < 1024 channels runs rms_norm_f32<256> (1024 threads from 1024
++// channels; that tree is reproduced too) (or the narrow-block
++// variant, documented bit-identical to it): thread t sums squares of columns t, t+256, ... (in that
++// order), each warp reduces its 32 partials with the xor butterfly, and the 8 warp sums plus 24 zero
++// partials are reduced by the same butterfly. Here one thread owns one spatial position (so loads
++// across a warp are contiguous) and evaluates exactly that reduction tree in registers.
++
++static bool norm_channel_enabled() {
++#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
++    return false;
++#else
++    static const bool enabled = [] {
++        const char * e = getenv("GGML_CUDA_NORM_CHANNEL");
++        return e == nullptr || atoi(e) != 0;
++    }();
++    return enabled && WARP_SIZE == 32;
++#endif
++}
++
++// GGML_CUDA_NORM_CHANNEL=2 fuses at every size (tests); by default only where it beats the unfused chain.
++static bool norm_channel_forced() {
++    static const bool forced = [] {
++        const char * e = getenv("GGML_CUDA_NORM_CHANNEL");
++        return e != nullptr && atoi(e) == 2;
++    }();
++    return forced;
++}
++
++static __device__ __forceinline__ float nc_butterfly32(float * v) {
++    // value every lane holds after warp_reduce_sum: lane l adds lane l^off at each step
++#pragma unroll
++    for (int off = 16; off > 0; off >>= 1) {
++#pragma unroll
++        for (int l = 0; l < off; ++l) {
++            v[l] = v[l] + v[l + off];
++        }
++    }
++    return v[0];
++}
++
++// BLOCK: the thread count of the rms_norm_f32 launch this reproduces (256 below 1024 columns, else 1024)
++template <int BLOCK>
++static __global__ void rms_norm_channel_f32(const float * x, const float * __restrict__ w, float * dst,
++                                            const int C, const int64_t P, const int64_t inner, const float eps) {
++    const int64_t q = (int64_t) blockIdx.x * blockDim.x + threadIdx.x;
++    if (q >= P) {
++        return;
++    }
++    // position q = (outer index, inner index); its channel c sits at base + c*inner
++    const int64_t p = (q / inner) * inner * C + q % inner;
++    const float * xp = x + p;
++    float wsum[32];
++#pragma unroll
++    for (int i = 0; i < 32; ++i) {
++        wsum[i] = 0.0f;
++    }
++#pragma unroll 1
++    for (int wi = 0; wi < BLOCK / 32; ++wi) {
++        float lane[32];
++#pragma unroll
++        for (int l = 0; l < 32; ++l) {
++            float tmp = 0.0f;
++            for (int col = wi * 32 + l; col < C; col += BLOCK) {
++                const float xi = xp[(int64_t) col * inner];
++                tmp += xi * xi;
++            }
++            lane[l] = tmp;
++        }
++        wsum[wi] = nc_butterfly32(lane);
++    }
++    const float tmp   = nc_butterfly32(wsum);
++    const float mean  = tmp / C;
++    const float scale = rsqrtf(mean + eps);
++    for (int col = 0; col < C; ++col) {
++        dst[p + (int64_t) col * inner] = scale * xp[(int64_t) col * inner] * w[col];
++    }
++}
++
++static bool nc_debug() {
++    static const bool on = getenv("GGML_CUDA_NORM_CHANNEL_DEBUG") != nullptr;
++    return on;
++}
++#define NC_REJECT(why) do { if (nc_debug()) { GGML_LOG_INFO("norm-channel: node %d (%s) rejected: %s\n", i, cont1->name, why); } return 0; } while (0)
++
++static bool is_view_op(const ggml_tensor * t) {
++    return t->op == GGML_OP_PERMUTE || t->op == GGML_OP_VIEW || t->op == GGML_OP_RESHAPE || t->op == GGML_OP_TRANSPOSE;
++}
++
++int ggml_cuda_try_fuse_rms_norm_channel(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i) {
++    if (!norm_channel_enabled()) {
++        return 0;
++    }
++    // node i: CONT whose source is a permuted view of a contiguous F32 x: cont[c, a, b, d] = x[a, b, d, c]
++    ggml_tensor * cont1 = cgraph->nodes[i];
++    if (cont1->op != GGML_OP_CONT || cont1->type != GGML_TYPE_F32) {
++        return 0;
++    }
++    const ggml_tensor * pv = cont1->src[0];
++    if (pv == nullptr || pv->op != GGML_OP_PERMUTE || pv->type != GGML_TYPE_F32) {
++        return 0;
++    }
++    const ggml_tensor * x = pv->src[0];
++    if (x == nullptr || x->type != GGML_TYPE_F32 || !ggml_is_contiguous(x) || pv->view_offs != 0 || pv->data != x->data) {
++        NC_REJECT("check 1");
++    }
++    // pv dim 0 (the normalised axis) must be one axis k of x; the other axes may come in any order
++    const int64_t C = pv->ne[0];
++    int k = -1;
++    for (int m = 0; m < GGML_MAX_DIMS; ++m) {
++        if (x->ne[m] == C && x->nb[m] == pv->nb[0] && (k < 0 || x->ne[m] > 1)) {
++            k = m;
++        }
++    }
++    // map every pv axis j > 0 of size > 1 to the x axis it reads
++    int pv_from_x[GGML_MAX_DIMS] = { -1, -1, -1, -1 };
++    pv_from_x[0] = k;
++    bool layout_ok = k >= 0 && C >= 1 && ggml_nelements(pv) == ggml_nelements(x);
++    for (int j = 1; j < GGML_MAX_DIMS && layout_ok; ++j) {
++        if (pv->ne[j] == 1) {
++            continue;
++        }
++        for (int m = 0; m < GGML_MAX_DIMS; ++m) {
++            if (m != k && x->ne[m] == pv->ne[j] && x->nb[m] == pv->nb[j]) {
++                pv_from_x[j] = m;
++            }
++        }
++        layout_ok = pv_from_x[j] >= 0;
++    }
++    if (!layout_ok) {
++        NC_REJECT("check 2");
++    }
++    const int64_t inner = x->nb[k] / (int64_t) sizeof(float);   // elements below the channel axis
++    const int64_t P     = ggml_nelements(x) / C;                 // positions normalised independently
++    // then RMS_NORM(cont1) and MUL(rms, w[C]) in that order (views in between are skipped)
++    auto next_op = [&](int from) {
++        for (int j = from + 1; j < cgraph->n_nodes; ++j) {
++            if (!is_view_op(cgraph->nodes[j]) || cgraph->nodes[j]->src[0] == cgraph->nodes[from]) {
++                return j;
++            }
++        }
++        return -1;
++    };
++    const int i_rms = next_op(i);
++    if (i_rms < 0) {
++        NC_REJECT("check 3");
++    }
++    ggml_tensor * rms = cgraph->nodes[i_rms];
++    const int i_mul = next_op(i_rms);
++    if (i_mul < 0) {
++        NC_REJECT("check 4");
++    }
++    ggml_tensor * mul = cgraph->nodes[i_mul];
++    if (rms->op != GGML_OP_RMS_NORM) {
++        if (nc_debug()) { GGML_LOG_INFO("norm-channel: C=%lld next op %s (%s)\n", (long long) C, ggml_op_desc(rms), rms->name); }
++        NC_REJECT("check 5a");
++    }
++    // the CONT must only make the permuted view contiguous (a CONT_4D may reshape it)
++    if (!ggml_are_same_shape(cont1, pv)) {
++        NC_REJECT("check 5e");
++    }
++    if (rms->src[0] != cont1) {
++        NC_REJECT("check 5b");
++    }
++    if (mul->op != GGML_OP_MUL || mul->src[0] != rms) {
++        if (nc_debug()) { GGML_LOG_INFO("norm-channel: C=%lld after rms %s (%s)\n", (long long) C, ggml_op_desc(mul), mul->name); }
++        NC_REJECT("check 5c");
++    }
++    if (!ggml_node_has_n_uses(cgraph, i, 1) || !ggml_node_has_n_uses(cgraph, i_rms, 1) || !ggml_node_has_n_uses(cgraph, i_mul, 1)) {
++        NC_REJECT("check 5d");
++    }
++    const ggml_tensor * w = mul->src[1];
++    if (w == nullptr || w->type != GGML_TYPE_F32 || !ggml_is_contiguous(w) || w->ne[0] != C || ggml_nelements(w) != C ||
++        mul->type != GGML_TYPE_F32 || !ggml_are_same_shape(mul, rms)) {
++        NC_REJECT("check 6");
++    }
++    // then PERMUTE(mul) back to x's layout and its CONT
++    const int i_pv2 = i_mul + 1;
++    if (i_pv2 + 1 >= cgraph->n_nodes) {
++        NC_REJECT("check 7");
++    }
++    ggml_tensor * pv2   = cgraph->nodes[i_pv2];
++    ggml_tensor * cont2 = cgraph->nodes[i_pv2 + 1];
++    bool back_ok = pv2->op == GGML_OP_PERMUTE && pv2->src[0] == mul && cont2->op == GGML_OP_CONT && cont2->src[0] == pv2 &&
++                   cont2->type == GGML_TYPE_F32 && ggml_are_same_shape(cont2, x) && ggml_are_same_shape(cont2, pv2) &&
++                   ggml_is_contiguous(cont2) && ggml_is_contiguous(mul);
++    // the graph does not count uses of views, so check by hand that cont2 is the only reader of pv2
++    for (int j = 0; j < cgraph->n_nodes && back_ok; ++j) {
++        const ggml_tensor * n = cgraph->nodes[j];
++        for (int s = 0; s < GGML_MAX_SRC && back_ok; ++s) {
++            back_ok = n == cont2 || n->src[s] != pv2;
++        }
++    }
++    // pv2 must be the inverse permutation: x axis m reads the mul axis that was built from x axis m
++    for (int j = 0; j < GGML_MAX_DIMS && back_ok; ++j) {
++        if (pv->ne[j] == 1 && j != 0) {
++            continue;
++        }
++        const int m = pv_from_x[j];
++        back_ok = x->ne[m] == 1 || pv2->nb[m] == mul->nb[j];
++    }
++    if (!back_ok) {
++        NC_REJECT("check 8");
++    }
++    // under ggml_build_forward_select only GGML_TENSOR_FLAG_COMPUTE nodes run: never fuse an inactive node
++    for (const ggml_tensor * n : { (const ggml_tensor *) cont1, (const ggml_tensor *) rms, (const ggml_tensor *) mul, (const ggml_tensor *) cont2 }) {
++        if ((n->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
++            NC_REJECT("check 8b");
++        }
++    }
++    // every node from i to the second CONT is part of the pattern or a view the loop skips
++    for (int j = i + 1; j < i_pv2 + 1; ++j) {
++        const ggml_tensor * n = cgraph->nodes[j];
++        if (j != i_rms && j != i_mul && j != i_pv2 && !is_view_op(n)) {
++            NC_REJECT("check 9");
++        }
++    }
++    // The result may reuse x's memory (x is often dead after this norm): with the same layout every
++    // thread reads all channels of its position before writing them, so only partial overlap is unsafe.
++    const char * x0 = (const char *) x->data;
++    const char * d0 = (const char *) cont2->data;
++    const bool partial_overlap = d0 != x0 && d0 < x0 + ggml_nbytes(x) && x0 < d0 + ggml_nbytes(cont2);
++    // Unfused, w is read by the MUL before the second CONT writes, so the allocator may hand w's buffer to
++    // cont2; the fused kernel reads w while writing cont2, so any overlap with w is unsafe.
++    const char * w0       = (const char *) w->data;
++    const bool w_overlap  = d0 < w0 + ggml_nbytes(w) && w0 < d0 + ggml_nbytes(cont2);
++    if (P > INT64_C(1) << 40 || partial_overlap || w_overlap) {
++        NC_REJECT("check 10");
++    }
++    // One thread per position walks every channel, so the kernel needs many positions to fill the GPU.
++    // Measured on B200 against CONT + RMS_NORM + MUL + CONT (Qwen-Image VAE shapes, whole image and tiles):
++    // with C < 1024 it wins from 16384 positions (173 vs 245 us), with C >= 1024 from 65536 (1.2 vs 3.3 ms),
++    // and loses below (P = 4096, C = 1152: 656 vs 209 us).
++    if (!norm_channel_forced() && P < (C < 1024 ? 16384 : 65536)) {
++        NC_REJECT("check 12");
++    }
++    float eps;
++    memcpy(&eps, rms->op_params, sizeof(float));
++    GGML_ASSERT(eps >= 0.0f);
++
++    const int threads = 128;
++    const int64_t blocks = (P + threads - 1) / threads;
++    if (blocks > INT_MAX) {
++        NC_REJECT("check 11");
++    }
++    if (C < 1024) {
++        rms_norm_channel_f32<256><<<(unsigned int) blocks, threads, 0, ctx.stream()>>>(
++            (const float *) x->data, (const float *) w->data, (float *) cont2->data, (int) C, P, inner, eps);
++    } else {
++        rms_norm_channel_f32<1024><<<(unsigned int) blocks, threads, 0, ctx.stream()>>>(
++            (const float *) x->data, (const float *) w->data, (float *) cont2->data, (int) C, P, inner, eps);
++    }
++    if (nc_debug()) {
++        GGML_LOG_INFO("norm-channel: fused C=%lld P=%lld\n", (long long) C, (long long) P);
++    }
++    return i_pv2 + 1 - i;
++}
+diff --git a/src/ggml-cuda/norm-channel.cuh b/src/ggml-cuda/norm-channel.cuh
+new file mode 100644
+index 00000000..da57e590
+--- /dev/null
++++ b/src/ggml-cuda/norm-channel.cuh
+@@ -0,0 +1,9 @@
++#pragma once
++#include "common.cuh"
++
++// CONT(PERMUTE(x)) -> RMS_NORM -> MUL(w) -> CONT(PERMUTE back): an RMS norm over the outermost
++// (channel) axis of a contiguous x, the way channel-first video VAEs (Wan) build it. Computes the
++// result straight from x into the final tensor, without the two relayout copies, bit-identical
++// to the unfused ops. Returns the number of extra nodes consumed (0: not this pattern / unsupported).
++// GGML_CUDA_NORM_CHANNEL=0 turns it off.
++int ggml_cuda_try_fuse_rms_norm_channel(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i);
+diff --git a/src/ggml-cuda/norm.cu b/src/ggml-cuda/norm.cu
+index 854d47b7..7ed01b7b 100644
+--- a/src/ggml-cuda/norm.cu
++++ b/src/ggml-cuda/norm.cu
+@@ -1,4 +1,5 @@
+ #include "norm.cuh"
++#include "group-norm-par.cuh"
+ #include <cstdint>
+ 
+ template <int block_size>
+@@ -567,6 +568,11 @@ void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst)
+     memcpy(&eps, dst->op_params + 1, sizeof(float));
+     GGML_ASSERT(eps >= 0.0f);
+ 
++    if (ggml_cuda_group_norm_par_supported(dst)) {
++        ggml_cuda_op_group_norm_par(ctx, dst, nullptr, nullptr, false, dst);
++        return;
++    }
++
+     int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups);
+     group_norm_f32_cuda(src0_d, dst_d, num_groups * src0->ne[3], eps, group_size, ggml_nelements(src0), stream);
+ }
+diff --git a/src/ggml-cuda/rope-pe.cu b/src/ggml-cuda/rope-pe.cu
+index b6dd7a69..2fe9a599 100644
+--- a/src/ggml-cuda/rope-pe.cu
++++ b/src/ggml-cuda/rope-pe.cu
+@@ -6,7 +6,7 @@
+ // Rounding matches the unfused graph exactly: __fmul_rn/__fadd_rn keep the two products and the
+ // sum as separately rounded operations (no FMA contraction), the scale is the same `scale*y + bias`
+ // expression the SCALE kernel evaluates, and the conversion uses the same ggml_cuda_cast as CPY.
+-template <typename dst_t, bool has_scale>
++template <typename dst_t, bool has_scale, bool interleaved = false>
+ static __global__ void rope_pe_permute_kernel(
+         const char * __restrict__ x, const float * __restrict__ pe, dst_t * __restrict__ dst,
+         const int D, const int H, const int64_t L, const int n_rot,
+@@ -21,9 +21,13 @@ static __global__ void rope_pe_permute_kernel(
+         int   j  = i;
+         float c0 = 0.0f;
+         float c1 = 0.0f;
++        int ja = i;
++        int jb = i;
+         if (rot) {
+-            const int r = i >= half;
+-            j           = i - r * half;
++            const int r = interleaved ? (i & 1) : (i >= half);
++            j           = interleaved ? (i >> 1) : i - r * half;
++            ja          = interleaved ? 2 * j : j;
++            jb          = interleaved ? 2 * j + 1 : j + half;
+             const float * t = pe + (l * half + j) * 4 + 2 * r;
+             c0 = t[0];
+             c1 = t[1];
+@@ -32,7 +36,7 @@ static __global__ void rope_pe_permute_kernel(
+             const float * xr = (const float *) (x + h * nb01 + l * nb02 + n * nb03);
+             float y;
+             if (rot) {
+-                y = __fadd_rn(__fmul_rn(xr[j], c0), __fmul_rn(xr[j + half], c1));
++                y = __fadd_rn(__fmul_rn(xr[ja], c0), __fmul_rn(xr[jb], c1));
+             } else {
+                 y = xr[i];
+             }
+@@ -48,7 +52,7 @@ static __global__ void rope_pe_permute_kernel(
+ // per (token, batch), the size and reduction rms_norm_f32_cuda's small-row path uses for this row
+ // length, so each normalised value is bit-identical to GGML_OP_RMS_NORM's; the normalised row is
+ // staged in shared memory for the RoPE partner reads.
+-template <typename dst_t, bool has_scale, int block_size>
++template <typename dst_t, bool has_scale, int block_size, bool interleaved = false>
+ static __global__ void rope_pe_permute_rms_kernel(
+         const char * __restrict__ x, const float * __restrict__ pe, dst_t * __restrict__ dst,
+         const int D, const int H, const int64_t L, const int n_rot,
+@@ -64,11 +68,15 @@ static __global__ void rope_pe_permute_rms_kernel(
+ 
+     const bool rot = i < n_rot;
+     int   j  = i;
++    int   ja = i;
++    int   jb = i;
+     float c0 = 0.0f;
+     float c1 = 0.0f;
+     if (rot) {
+-        const int r = i >= half;
+-        j           = i - r * half;
++        const int r = interleaved ? (i & 1) : (i >= half);
++        j           = interleaved ? (i >> 1) : i - r * half;
++        ja          = interleaved ? 2 * j : j;
++        jb          = interleaved ? 2 * j + 1 : j + half;
+         const float * t = pe + (l * half + j) * 4 + 2 * r;
+         c0 = t[0];
+         c1 = t[1];
+@@ -91,7 +99,7 @@ static __global__ void rope_pe_permute_rms_kernel(
+         if (i < D) {
+             float y;
+             if (rot) {
+-                y = __fadd_rn(__fmul_rn(xs[j], c0), __fmul_rn(xs[j + half], c1));
++                y = __fadd_rn(__fmul_rn(xs[ja], c0), __fmul_rn(xs[jb], c1));
+             } else {
+                 y = xs[i];
+             }
+@@ -117,7 +125,7 @@ static bool rope_pe_rms_small_rows_enabled() {
+ #endif
+ }
+ 
+-template <typename dst_t, int block_size>
++template <typename dst_t, int block_size, bool interleaved>
+ static void rope_pe_permute_rms_launch(const ggml_tensor * src0, const float * pe, dst_t * dst, const int n_rot,
+                                        const float scale, const float eps, cudaStream_t stream) {
+     const int     D = (int) src0->ne[0];
+@@ -128,29 +136,29 @@ static void rope_pe_permute_rms_launch(const ggml_tensor * src0, const float * p
+     const dim3 block(block_size, 1, 1);
+     const char * x = (const char *) src0->data;
+     if (scale != 1.0f) {
+-        rope_pe_permute_rms_kernel<dst_t, true, block_size><<<grid, block, 0, stream>>>(
++        rope_pe_permute_rms_kernel<dst_t, true, block_size, interleaved><<<grid, block, 0, stream>>>(
+             x, pe, dst, D, H, L, n_rot, src0->nb[1], src0->nb[2], src0->nb[3], scale, 0.0f, eps);
+     } else {
+-        rope_pe_permute_rms_kernel<dst_t, false, block_size><<<grid, block, 0, stream>>>(
++        rope_pe_permute_rms_kernel<dst_t, false, block_size, interleaved><<<grid, block, 0, stream>>>(
+             x, pe, dst, D, H, L, n_rot, src0->nb[1], src0->nb[2], src0->nb[3], scale, 0.0f, eps);
+     }
+ }
+ 
+-template <typename dst_t>
++template <typename dst_t, bool interleaved = false>
+ static void rope_pe_permute_rms_cuda(const ggml_tensor * src0, const float * pe, dst_t * dst, const int n_rot,
+                                      const float scale, const float eps, cudaStream_t stream) {
+     const int D = (int) src0->ne[0];
+     GGML_ASSERT(D <= 128);
+     if (D <= 32) {
+-        rope_pe_permute_rms_launch<dst_t, 32>(src0, pe, dst, n_rot, scale, eps, stream);
++        rope_pe_permute_rms_launch<dst_t, 32, interleaved>(src0, pe, dst, n_rot, scale, eps, stream);
+     } else if (D <= 64) {
+-        rope_pe_permute_rms_launch<dst_t, 64>(src0, pe, dst, n_rot, scale, eps, stream);
++        rope_pe_permute_rms_launch<dst_t, 64, interleaved>(src0, pe, dst, n_rot, scale, eps, stream);
+     } else {
+-        rope_pe_permute_rms_launch<dst_t, 128>(src0, pe, dst, n_rot, scale, eps, stream);
++        rope_pe_permute_rms_launch<dst_t, 128, interleaved>(src0, pe, dst, n_rot, scale, eps, stream);
+     }
+ }
+ 
+-template <typename dst_t>
++template <typename dst_t, bool interleaved = false>
+ static void rope_pe_permute_cuda(const ggml_tensor * src0, const float * pe, dst_t * dst, const int n_rot,
+                                  const float scale, cudaStream_t stream) {
+     const int     D = (int) src0->ne[0];
+@@ -162,10 +170,10 @@ static void rope_pe_permute_cuda(const ggml_tensor * src0, const float * pe, dst
+     const dim3 block(D >= 128 ? 128 : ((D + 31) / 32) * 32, 1, 1);
+     const char * x = (const char *) src0->data;
+     if (scale != 1.0f) {
+-        rope_pe_permute_kernel<dst_t, true><<<grid, block, 0, stream>>>(
++        rope_pe_permute_kernel<dst_t, true, interleaved><<<grid, block, 0, stream>>>(
+             x, pe, dst, D, H, L, n_rot, src0->nb[1], src0->nb[2], src0->nb[3], scale, 0.0f);
+     } else {
+-        rope_pe_permute_kernel<dst_t, false><<<grid, block, 0, stream>>>(
++        rope_pe_permute_kernel<dst_t, false, interleaved><<<grid, block, 0, stream>>>(
+             x, pe, dst, D, H, L, n_rot, src0->nb[1], src0->nb[2], src0->nb[3], scale, 0.0f);
+     }
+ }
+@@ -184,22 +192,41 @@ void ggml_cuda_op_rope_pe_permute(ggml_backend_cuda_context & ctx, ggml_tensor *
+     cudaStream_t  stream = ctx.stream();
+     const float * pe_d   = pe ? (const float *) pe->data : nullptr;
+ 
++    // interleaved pairing (2j, 2j+1) only matters when something rotates
++    const bool interleaved = n_rot > 0 && ggml_get_op_params_i32(dst, 4) != 0;
++
+     if (ggml_get_op_params_i32(dst, 2) != 0) {
+         const float eps = ggml_get_op_params_f32(dst, 3);
+         if (dst->type == GGML_TYPE_F16) {
+-            rope_pe_permute_rms_cuda(src0, pe_d, (half *) dst->data, n_rot, scale, eps, stream);
++            if (interleaved) {
++                rope_pe_permute_rms_cuda<half, true>(src0, pe_d, (half *) dst->data, n_rot, scale, eps, stream);
++            } else {
++                rope_pe_permute_rms_cuda<half, false>(src0, pe_d, (half *) dst->data, n_rot, scale, eps, stream);
++            }
+         } else {
+             GGML_ASSERT(dst->type == GGML_TYPE_F32);
+-            rope_pe_permute_rms_cuda(src0, pe_d, (float *) dst->data, n_rot, scale, eps, stream);
++            if (interleaved) {
++                rope_pe_permute_rms_cuda<float, true>(src0, pe_d, (float *) dst->data, n_rot, scale, eps, stream);
++            } else {
++                rope_pe_permute_rms_cuda<float, false>(src0, pe_d, (float *) dst->data, n_rot, scale, eps, stream);
++            }
+         }
+         return;
+     }
+ 
+     if (dst->type == GGML_TYPE_F16) {
+-        rope_pe_permute_cuda(src0, pe_d, (half *) dst->data, n_rot, scale, stream);
++        if (interleaved) {
++            rope_pe_permute_cuda<half, true>(src0, pe_d, (half *) dst->data, n_rot, scale, stream);
++        } else {
++            rope_pe_permute_cuda<half, false>(src0, pe_d, (half *) dst->data, n_rot, scale, stream);
++        }
+     } else {
+         GGML_ASSERT(dst->type == GGML_TYPE_F32);
+-        rope_pe_permute_cuda(src0, pe_d, (float *) dst->data, n_rot, scale, stream);
++        if (interleaved) {
++            rope_pe_permute_cuda<float, true>(src0, pe_d, (float *) dst->data, n_rot, scale, stream);
++        } else {
++            rope_pe_permute_cuda<float, false>(src0, pe_d, (float *) dst->data, n_rot, scale, stream);
++        }
+     }
+ }
+ 
+diff --git a/src/ggml-cuda/rope.cu b/src/ggml-cuda/rope.cu
+index 33cee75a..ef25d68f 100644
+--- a/src/ggml-cuda/rope.cu
++++ b/src/ggml-cuda/rope.cu
+@@ -13,6 +13,9 @@ static __global__ void rope_apply_f32(
+         const float * x, const float * pe, float * dst,
+         int64_t half_dim, int64_t heads, int64_t tokens, int64_t pairs,
+         RopeApplyStrides strides) {
++    // Launched with programmatic dependent launch (ggml_cuda_kernel_launch): wait for the producer of x
++    // before reading it. Without this the kernel could read q/k while the preceding norm still wrote them.
++    ggml_cuda_pdl_sync();
+     const int64_t index = (int64_t) blockIdx.x*blockDim.x + threadIdx.x;
+     if (index >= pairs) {
+         return;
+diff --git a/src/ggml.c b/src/ggml.c
+index 1dcc8a0f..ef3c750f 100644
+--- a/src/ggml.c
++++ b/src/ggml.c
+@@ -5773,6 +5773,14 @@ struct ggml_tensor * ggml_rope_pe_permute_rms(
+     return result;
+ }
+ 
++struct ggml_tensor * ggml_rope_pe_permute_set_interleaved(
++        struct ggml_tensor  * t,
++        bool                  interleaved) {
++    GGML_ASSERT(t->op == GGML_OP_ROPE_PE_PERMUTE);
++    ggml_set_op_params_i32(t, 4, interleaved ? 1 : 0);
++    return t;
++}
++
+ static struct ggml_tensor * ggml_modulate_rows_impl(
+         struct ggml_context * ctx,
+         struct ggml_tensor  * acc,
diff --git a/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch
new file mode 100644
index 0000000000..b3e8b9a812
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0008-ggml-cpu-avx512-variants-need-avx-vnni.patch
@@ -0,0 +1,15 @@
+diff --git a/src/ggml-cpu/arch/x86/cpu-feats.cpp b/src/ggml-cpu/arch/x86/cpu-feats.cpp
+index d775a036..a29335b9 100644
+--- a/src/ggml-cpu/arch/x86/cpu-feats.cpp
++++ b/src/ggml-cpu/arch/x86/cpu-feats.cpp
+@@ -300,6 +300,10 @@ static int ggml_backend_cpu_x86_score() {
+     if (!is.AVX512VL()) { return 0; }
+     if (!is.AVX512DQ()) { return 0; }
+     if (!is.AVX512BW()) { return 0; }
++    // Unsloth: the AVX512 variants are only offered on CPUs that also have AVX-VNNI (Zen 5,
++    // Sapphire Rapids and newer). On Zen 4, which has AVX512 but not AVX-VNNI, they ran
++    // stable-diffusion.cpp 15 to 50% slower than the AVX2 variant; the AVX2 one is picked there.
++    if (!is.AVX_VNNI()) { return 0; }
+     score += 1<<7;
+ #endif
+ #ifdef GGML_AVX512_VBMI
diff --git a/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch b/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch
new file mode 100644
index 0000000000..291d134825
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0009-ggml-backend-no-cwd-search.patch
@@ -0,0 +1,70 @@
+diff --git a/src/ggml-backend-reg.cpp b/src/ggml-backend-reg.cpp
+index 4e9681c9..a92d21a5 100644
+--- a/src/ggml-backend-reg.cpp
++++ b/src/ggml-backend-reg.cpp
+@@ -418,7 +418,8 @@ static fs::path get_executable_path() {
+     }
+     return base_path + "/";
+ #elif defined(__linux__) || defined(__FreeBSD__)
+-    std::string base_path = ".";
++    // Unsloth: no "." fallback; on failure return empty so the caller skips it (not the cwd).
++    std::string base_path;
+     std::vector<char> path(1024);
+     while (true) {
+         // get executable path
+@@ -428,7 +429,7 @@ static fs::path get_executable_path() {
+         ssize_t len = readlink("/proc/curproc/file", path.data(), path.size());
+ #    endif
+         if (len == -1) {
+-            break;
++            return {};
+         }
+         if (len < (ssize_t) path.size()) {
+             base_path = std::string(path.data(), len);
+@@ -444,10 +445,21 @@ static fs::path get_executable_path() {
+ 
+     return base_path + "/";
+ #elif defined(_WIN32)
++    // Unsloth: grow the buffer; a truncated result (len == size) would yield an ancestor directory.
+     std::vector<wchar_t> path(MAX_PATH);
+-    DWORD len = GetModuleFileNameW(NULL, path.data(), path.size());
+-    if (len == 0) {
+-        return {};
++    DWORD len;
++    while (true) {
++        len = GetModuleFileNameW(NULL, path.data(), (DWORD) path.size());
++        if (len == 0) {
++            return {};
++        }
++        if (len < path.size()) {
++            break;
++        }
++        if (path.size() >= 32768) {
++            return {};
++        }
++        path.resize(path.size() * 2);
+     }
+     std::wstring base_path(path.data(), len);
+     // remove executable name
+@@ -488,14 +500,13 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
+ #ifdef GGML_BACKEND_DIR
+         search_paths.push_back(fs::u8path(GGML_BACKEND_DIR));
+ #endif
+-        // default search paths: executable directory, current directory
+-        search_paths.push_back(get_executable_path());
+-        std::error_code cwd_ec;
+-        const fs::path cwd = fs::current_path(cwd_ec);
+-        if (cwd_ec) {
+-            GGML_LOG_DEBUG("%s: failed to get current directory: %s\n", __func__, cwd_ec.message().c_str());
+-        } else {
+-            search_paths.push_back(cwd);
++        // Unsloth: default search path is the executable directory only. Every matching library is
++        // loaded (running its constructors) before it is scored, so searching the current directory
++        // would run any libggml-<name>-*.so placed wherever the user launches sd-cli / sd-server.
++        // Use GGML_BACKEND_DIR or an explicit path to load backends from elsewhere.
++        const fs::path executable_dir = get_executable_path();
++        if (!executable_dir.empty()) {
++            search_paths.push_back(executable_dir);
+         }
+     } else {
+         search_paths.push_back(fs::u8path(user_search_path));
diff --git a/scripts/unsloth/ggml-patches/0010-ggml-metal-rope-apply.patch b/scripts/unsloth/ggml-patches/0010-ggml-metal-rope-apply.patch
new file mode 100644
index 0000000000..280996ed28
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0010-ggml-metal-rope-apply.patch
@@ -0,0 +1,205 @@
+diff --git a/src/ggml-metal/ggml-metal-device.cpp b/src/ggml-metal/ggml-metal-device.cpp
+index 2c7972b0..1d2e0318 100644
+--- a/src/ggml-metal/ggml-metal-device.cpp
++++ b/src/ggml-metal/ggml-metal-device.cpp
+@@ -2564,6 +2564,20 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_roll(ggml_metal_
+     return res;
+ }
+ 
++ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope_apply(ggml_metal_library_t lib, const ggml_tensor * op) {
++    assert(op->op == GGML_OP_ROPE_APPLY);
++    GGML_UNUSED(op);
++
++    const char * name = "kernel_rope_apply_f32";
++
++    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
++    if (!res.pipeline) {
++        res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
++    }
++
++    return res;
++}
++
+ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad(ggml_metal_library_t lib, const ggml_tensor * op) {
+     assert(op->op == GGML_OP_PAD);
+ 
+diff --git a/src/ggml-metal/ggml-metal-device.h b/src/ggml-metal/ggml-metal-device.h
+index 4012b9d5..48d77221 100644
+--- a/src/ggml-metal/ggml-metal-device.h
++++ b/src/ggml-metal/ggml-metal-device.h
+@@ -173,6 +173,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale
+ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad               (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_pad_reflect_1d    (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_roll              (ggml_metal_library_t lib, const struct ggml_tensor * op);
++struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope_apply        (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_arange            (ggml_metal_library_t lib, const struct ggml_tensor * op);
+ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_timestep_embedding(ggml_metal_library_t lib, const struct ggml_tensor * op);
+ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_opt_step_adamw    (ggml_metal_library_t lib, const struct ggml_tensor * op);
+diff --git a/src/ggml-metal/ggml-metal-device.m b/src/ggml-metal/ggml-metal-device.m
+index 44519903..8634d284 100644
+--- a/src/ggml-metal/ggml-metal-device.m
++++ b/src/ggml-metal/ggml-metal-device.m
+@@ -1695,6 +1695,10 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
+         case GGML_OP_ROPE:
+         case GGML_OP_ROPE_BACK:
+             return true;
++        case GGML_OP_ROPE_APPLY:
++            return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 &&
++                   op->type == GGML_TYPE_F32 && ggml_is_contiguous(op) &&
++                   op->src[0]->ne[1] <= 65535 && op->src[0]->ne[3] <= 65535;
+         case GGML_OP_IM2COL:
+             return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
+         case GGML_OP_CONV_2D:
+diff --git a/src/ggml-metal/ggml-metal-impl.h b/src/ggml-metal/ggml-metal-impl.h
+index 4ffb5a77..a358e659 100644
+--- a/src/ggml-metal/ggml-metal-impl.h
++++ b/src/ggml-metal/ggml-metal-impl.h
+@@ -1177,6 +1177,22 @@ typedef struct {
+     int32_t  s3;
+ } ggml_metal_kargs_roll;
+ 
++typedef struct {
++    int32_t  ne00;
++    int32_t  ne01;
++    int32_t  ne02;
++    int32_t  ne03;
++    uint64_t nb00;
++    uint64_t nb01;
++    uint64_t nb02;
++    uint64_t nb03;
++    uint64_t nb10;
++    uint64_t nb11;
++    uint64_t nb12;
++    uint64_t nb13;
++    int32_t  interleaved;
++} ggml_metal_kargs_rope_apply;
++
+ typedef struct {
+     uint64_t nb1;
+     int      dim;
+diff --git a/src/ggml-metal/ggml-metal-ops.cpp b/src/ggml-metal/ggml-metal-ops.cpp
+index 53c98161..551d62b0 100644
+--- a/src/ggml-metal/ggml-metal-ops.cpp
++++ b/src/ggml-metal/ggml-metal-ops.cpp
+@@ -414,6 +414,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
+             {
+                 n_fuse = ggml_metal_op_rope(ctx, idx);
+             } break;
++        case GGML_OP_ROPE_APPLY:
++            {
++                n_fuse = ggml_metal_op_rope_apply(ctx, idx);
++            } break;
+         case GGML_OP_IM2COL:
+             {
+                 n_fuse = ggml_metal_op_im2col(ctx, idx);
+@@ -5257,6 +5261,47 @@ int ggml_metal_op_roll(ggml_metal_op_t ctx, int idx) {
+     return 1;
+ }
+ 
++int ggml_metal_op_rope_apply(ggml_metal_op_t ctx, int idx) {
++    ggml_tensor * op = ctx->node(idx);
++
++    ggml_metal_library_t lib = ctx->lib;
++    ggml_metal_encoder_t enc = ctx->enc;
++
++    GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne);
++    GGML_TENSOR_LOCALS(uint64_t, nb0, op->src[0], nb);
++    GGML_TENSOR_LOCALS(uint64_t, nb1, op->src[1], nb);
++
++    ggml_metal_kargs_rope_apply args = {
++        /*.ne00        =*/ ne00,
++        /*.ne01        =*/ ne01,
++        /*.ne02        =*/ ne02,
++        /*.ne03        =*/ ne03,
++        /*.nb00        =*/ nb00,
++        /*.nb01        =*/ nb01,
++        /*.nb02        =*/ nb02,
++        /*.nb03        =*/ nb03,
++        /*.nb10        =*/ nb10,
++        /*.nb11        =*/ nb11,
++        /*.nb12        =*/ nb12,
++        /*.nb13        =*/ nb13,
++        /*.interleaved =*/ ggml_get_op_params_i32(op, 0) != 0 ? 1 : 0,
++    };
++
++    auto pipeline = ggml_metal_library_get_pipeline_rope_apply(lib, op);
++
++    const int nth = std::max(1, std::min(ne00/2, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)));
++
++    ggml_metal_encoder_set_pipeline(enc, pipeline);
++    ggml_metal_encoder_set_bytes   (enc, &args, sizeof(args), 0);
++    ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
++    ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
++    ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op),         3);
++
++    ggml_metal_encoder_dispatch_threadgroups(enc, ne02, ne01, ne03, nth, 1, 1);
++
++    return 1;
++}
++
+ int ggml_metal_op_pad(ggml_metal_op_t ctx, int idx) {
+     ggml_tensor * op = ctx->node(idx);
+ 
+diff --git a/src/ggml-metal/ggml-metal-ops.h b/src/ggml-metal/ggml-metal-ops.h
+index 583d1156..ef6be30f 100644
+--- a/src/ggml-metal/ggml-metal-ops.h
++++ b/src/ggml-metal/ggml-metal-ops.h
+@@ -93,6 +93,7 @@ int ggml_metal_op_upscale           (ggml_metal_op_t ctx, int idx);
+ int ggml_metal_op_pad               (ggml_metal_op_t ctx, int idx);
+ int ggml_metal_op_pad_reflect_1d    (ggml_metal_op_t ctx, int idx);
+ int ggml_metal_op_roll              (ggml_metal_op_t ctx, int idx);
++int ggml_metal_op_rope_apply        (ggml_metal_op_t ctx, int idx);
+ int ggml_metal_op_arange            (ggml_metal_op_t ctx, int idx);
+ int ggml_metal_op_timestep_embedding(ggml_metal_op_t ctx, int idx);
+ int ggml_metal_op_argmax            (ggml_metal_op_t ctx, int idx);
+diff --git a/src/ggml-metal/kernels/rope.metal b/src/ggml-metal/kernels/rope.metal
+index 401ceacb..b47563c6 100644
+--- a/src/ggml-metal/kernels/rope.metal
++++ b/src/ggml-metal/kernels/rope.metal
+@@ -331,3 +331,46 @@ template [[host_name("kernel_rope_multi_f16")]] kernel kernel_rope_multi_t kerne
+ 
+ template [[host_name("kernel_rope_vision_f32")]] kernel kernel_rope_vision_t kernel_rope_vision<float>;
+ template [[host_name("kernel_rope_vision_f16")]] kernel kernel_rope_vision_t kernel_rope_vision<half>;
++
++// x [d, n_head, L, N] (any strides), pe [2, 2, d/2, L] of [[cos, -sin], [sin, cos]]; dst contiguous [d, L, n_head*N]
++kernel void kernel_rope_apply_f32(
++        constant ggml_metal_kargs_rope_apply & args,
++        device const char * src0,
++        device const char * src1,
++        device       char * dst,
++        uint3 tgpig[[threadgroup_position_in_grid]],
++        uint3 tpitg[[thread_position_in_threadgroup]],
++        uint3   ntg[[threads_per_threadgroup]]) {
++    const int64_t token = tgpig.x;
++    const int64_t head  = tgpig.y;
++    const int64_t batch = tgpig.z;
++
++    const int64_t half_dim = args.ne00/2;
++
++    device const char * x_row  = src0 + head*args.nb01 + token*args.nb02 + batch*args.nb03;
++    device const char * pe_row = src1 + token*args.nb13;
++    device       float * out   = (device float *) dst + (token + args.ne02*(head + args.ne01*batch))*args.ne00;
++
++    for (int64_t j = tpitg.x; j < half_dim; j += ntg.x) {
++        const int64_t i0 = args.interleaved ? 2*j     : j;
++        const int64_t i1 = args.interleaved ? 2*j + 1 : j + half_dim;
++
++        const float a = *(device const float *) (x_row + i0*args.nb00);
++        const float b = *(device const float *) (x_row + i1*args.nb00);
++
++        device const char * m = pe_row + j*args.nb12;
++
++        const float c = *(device const float *) (m);
++        const float s = *(device const float *) (m + args.nb11);
++
++        if (args.interleaved) {
++            const float neg_s = *(device const float *) (m + args.nb10);
++            const float c1    = *(device const float *) (m + args.nb10 + args.nb11);
++            out[i0] = a*c + b*neg_s;
++            out[i1] = a*s + b*c1;
++        } else {
++            out[i0] = a*c - b*s;
++            out[i1] = a*s + b*c;
++        }
++    }
++}
diff --git a/scripts/unsloth/ggml-patches/0011-ggml-metal-no-fp8-int8-mul-mat.patch b/scripts/unsloth/ggml-patches/0011-ggml-metal-no-fp8-int8-mul-mat.patch
new file mode 100644
index 0000000000..0b44b86fe6
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0011-ggml-metal-no-fp8-int8-mul-mat.patch
@@ -0,0 +1,18 @@
+diff --git a/src/ggml-metal/ggml-metal-device.m b/src/ggml-metal/ggml-metal-device.m
+index 8634d284..1dc0d248 100644
+--- a/src/ggml-metal/ggml-metal-device.m
++++ b/src/ggml-metal/ggml-metal-device.m
+@@ -1521,6 +1521,13 @@ static bool ggml_metal_supports_mul_mat_op(
+         return false;
+     }
+ 
++    // no FP8/INT8 mul_mv/mul_mm kernels: claiming support aborts at encode instead of letting sd.cpp cast or the op fall back
++    if (op->src[0]->type == GGML_TYPE_F8_E4M3 ||
++        op->src[0]->type == GGML_TYPE_F8_E5M2 ||
++        op->src[0]->type == GGML_TYPE_I8) {
++        return false;
++    }
++
+     if (op->src[1]->type != GGML_TYPE_F16) {
+         return true;
+     }
diff --git a/scripts/unsloth/ggml-patches/0012-ggml-metal-group-norm-threads.patch b/scripts/unsloth/ggml-patches/0012-ggml-metal-group-norm-threads.patch
new file mode 100644
index 0000000000..b65cbdddd9
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0012-ggml-metal-group-norm-threads.patch
@@ -0,0 +1,61 @@
+diff --git a/src/ggml-metal/ggml-metal-ops.cpp b/src/ggml-metal/ggml-metal-ops.cpp
+index 551d62b0..1aa237bc 100644
+--- a/src/ggml-metal/ggml-metal-ops.cpp
++++ b/src/ggml-metal/ggml-metal-ops.cpp
+@@ -4383,13 +4383,13 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) {
+ 
+     auto pipeline = ggml_metal_library_get_pipeline_group_norm(lib, op);
+ 
+-    int nth = 32; // SIMD width
+-    //while (nth < ne00/4 && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
+-    //    nth *= 2;
+-    //}
++    // one simdgroup per group starves the GPU on VAE group norms; the 32-float reduction buffer caps nth at 1024
++    const int64_t gs = (int64_t) ne00*ne01*((ne02 + ngrp - 1)/ngrp);
+ 
+-    //nth = std::min(nth, ggml_metal_pipeline_max_theads_per_threadgroup(pipeline));
+-    //nth = std::min(nth, ne00/4);
++    int nth = 32; // SIMD width
++    while (nth < 1024 && (int64_t) nth*8 <= gs && nth < ggml_metal_pipeline_max_theads_per_threadgroup(pipeline)) {
++        nth *= 2;
++    }
+ 
+     const size_t smem = pipeline.smem;
+ 
+@@ -4400,7 +4400,7 @@ int ggml_metal_op_group_norm(ggml_metal_op_t ctx, int idx) {
+ 
+     ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
+ 
+-    ggml_metal_encoder_dispatch_threadgroups(enc, ngrp, 1, 1, nth, 1, 1);
++    ggml_metal_encoder_dispatch_threadgroups(enc, ngrp*ne03, 1, 1, nth, 1, 1);
+ 
+     return 1;
+ }
+diff --git a/src/ggml-metal/kernels/norm.metal b/src/ggml-metal/kernels/norm.metal
+index c76d0c7f..a301bbf0 100644
+--- a/src/ggml-metal/kernels/norm.metal
++++ b/src/ggml-metal/kernels/norm.metal
+@@ -249,7 +249,12 @@ kernel void kernel_group_norm_f32(
+     const int64_t ne = args.ne00*args.ne01*args.ne02;
+     const int64_t gs = args.ne00*args.ne01*((args.ne02 + args.ngrp - 1) / args.ngrp);
+ 
+-    int start = tgpig * gs;
++    // src0 is contiguous
++    const int64_t i03 = tgpig / args.ngrp;
++    src0 += i03*ne;
++    dst  += i03*ne;
++
++    int start = (tgpig % args.ngrp) * gs;
+     int end   = start + gs;
+ 
+     start += tpitg;
+@@ -294,6 +299,9 @@ kernel void kernel_group_norm_f32(
+ 
+     tmp = simd_sum(tmp);
+     if (ntg > N_SIMDWIDTH) {
++        // every simdgroup must have read the mean reduction from buf before it is reused
++        threadgroup_barrier(mem_flags::mem_threadgroup);
++
+         if (sgitg == 0) {
+             buf[tiisg] = 0.0f;
+         }
diff --git a/scripts/unsloth/ggml-patches/0013-ggml-sched-inplace-op-owner-backend.patch b/scripts/unsloth/ggml-patches/0013-ggml-sched-inplace-op-owner-backend.patch
new file mode 100644
index 0000000000..fe8b11351f
--- /dev/null
+++ b/scripts/unsloth/ggml-patches/0013-ggml-sched-inplace-op-owner-backend.patch
@@ -0,0 +1,32 @@
+diff --git a/src/ggml-backend.cpp b/src/ggml-backend.cpp
+index 48802d68..a1cfe7da 100644
+--- a/src/ggml-backend.cpp
++++ b/src/ggml-backend.cpp
+@@ -1442,6 +1442,27 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
+         GGML_ASSERT(*cur_backend_id != -1);
+     }
+ 
++    // pass 4.5: an in-place op writes into its view_src's memory, so run it on the backend owning
++    // that buffer; elsewhere it gets a foreign pointer (Metal "buffer is nil", CUDA illegal address)
++    for (int i = 0; i < graph->n_nodes; i++) {
++        struct ggml_tensor * node = graph->nodes[i];
++        if (node->view_src == NULL || ggml_is_view_op(node->op)) {
++            continue;
++        }
++        int * cur_backend_id = &tensor_backend_id(node);
++        const int owner_id = node->view_src->buffer != NULL
++            ? ggml_backend_sched_backend_from_buffer(sched, node->view_src, node)
++            : tensor_backend_id(node->view_src);
++        if (owner_id == -1 || owner_id == *cur_backend_id ||
++            ggml_backend_sched_buffer_supported(sched, node->view_src, *cur_backend_id)) {
++            continue;
++        }
++        if (ggml_backend_supports_op(sched->backends[owner_id], node)) {
++            *cur_backend_id = owner_id;
++            SET_CAUSE(node, "4.5.inplace");
++        }
++    }
++
+     // pass 5: split graph, find tensors that need to be copied
+     {
+         int i_split = 0;
diff --git a/scripts/unsloth/package_bundle.py b/scripts/unsloth/package_bundle.py
index 6c87db4668..ecc93ae63f 100755
--- a/scripts/unsloth/package_bundle.py
+++ b/scripts/unsloth/package_bundle.py
@@ -16,6 +16,8 @@
     COMMIT    source commit SHA (provenance)
     SOURCE_REPO   e.g. leejet/stable-diffusion.cpp
     LICENSE_FILE  path to the LICENSE to include (optional)
+    KEEP_LAYOUT   "1" ships every file under BIN_DIR at its relative path instead of
+                  flattening (the ROCm leg stages lib/ and .kpack/ that must stay siblings)
 
 The zip unpacks into a single named dir ``sd-<TAG>-bin-<LABEL>/`` containing the
 binaries, their sibling runtime libs, LICENSE, and an UNSLOTH_BUILD.txt provenance
@@ -27,6 +29,7 @@
 
 import datetime as _dt
 import os
+import re
 import sys
 import zipfile
 from pathlib import Path
@@ -57,16 +60,30 @@ def _is_runtime_lib(name: str) -> bool:
     return ".so." in lowered
 
 
-def _collect(bin_dir: Path) -> list[Path]:
-    """The binaries + sibling runtime libs to ship. Recurse so a nested bin/ layout
-    (some generators emit build/bin/, some build/bin/Release/) is still captured."""
-    found: list[Path] = []
-    for p in sorted(bin_dir.rglob("*")):
-        if not p.is_file():
-            continue
-        if p.name in _BINARIES or _is_runtime_lib(p.name):
-            found.append(p)
-    return found
+# libggml-cuda.so.0, the name DT_NEEDED spells. The fully versioned file behind it and the bare
+# libggml-cuda.so link are build-time names only.
+_SONAME_RE = re.compile(r"lib.+\.so\.\d+")
+
+
+def _collect(bin_dir: Path, keep_layout: bool = False) -> list[tuple[str, Path]]:
+    """(name in the bundle, file) for the binaries + sibling runtime libs. Recurse so a nested bin/
+    layout (some generators emit build/bin/, some build/bin/Release/) is still captured.
+
+    A shared-ggml build leaves each library as a real file plus two symlinks, and zipfile follows
+    links, so shipping all three stores the 100+ MB CUDA library three times. Ship it once, under
+    its soname. With keep_layout, every file ships at its path under bin_dir."""
+    if keep_layout:
+        return [(p.relative_to(bin_dir).as_posix(), p) for p in sorted(bin_dir.rglob("*")) if p.is_file()]
+    wanted = [
+        p for p in sorted(bin_dir.rglob("*"))
+        if p.is_file() and (p.name in _BINARIES or _is_runtime_lib(p.name))
+    ]
+    found = {p.name: p.resolve() for p in wanted if p.is_symlink() and _SONAME_RE.fullmatch(p.name)}
+    under_soname = set(found.values())
+    for p in wanted:
+        if p.name not in found and p.resolve() not in under_soname:
+            found[p.name] = p
+    return sorted(found.items())
 
 
 def main() -> int:
@@ -82,12 +99,12 @@ def main() -> int:
         print(f"package_bundle: BIN_DIR {bin_dir} is not a directory", file = sys.stderr)
         return 2
 
-    files = _collect(bin_dir)
-    have_cli = any(f.name in ("sd-cli", "sd-cli.exe") for f in files)
+    files = _collect(bin_dir, os.environ.get("KEEP_LAYOUT", "").strip() == "1")
+    have_cli = any(Path(name).name in ("sd-cli", "sd-cli.exe") for name, _ in files)
     if not have_cli:
         print(f"package_bundle: no sd-cli under {bin_dir}; refusing to package", file = sys.stderr)
         return 1
-    have_server = any(f.name in ("sd-server", "sd-server.exe") for f in files)
+    have_server = any(Path(name).name in ("sd-server", "sd-server.exe") for name, _ in files)
 
     out_dir.mkdir(parents = True, exist_ok = True)
     stem = f"sd-{tag}-bin-{label}"
@@ -108,10 +125,10 @@ def main() -> int:
     # Deterministic-ish: sort members; drop the archive if it already exists.
     zip_path.unlink(missing_ok = True)
     with zipfile.ZipFile(zip_path, "w", compression = zipfile.ZIP_DEFLATED) as zf:
-        for f in files:
-            # Flatten under the named top-level dir; keep just the basename so the
-            # binaries sit at sd-<tag>-bin-<label>/<name> regardless of build layout.
-            zf.write(f, arcname = f"{stem}/{f.name}")
+        for name, f in files:
+            # Under sd-<tag>-bin-<label>/: flat (binaries at <stem>/<name> regardless of build
+            # layout) unless KEEP_LAYOUT.
+            zf.write(f, arcname = f"{stem}/{name}")
         zf.writestr(f"{stem}/UNSLOTH_BUILD.txt", provenance)
         if license_file and Path(license_file).is_file():
             zf.write(license_file, arcname = f"{stem}/LICENSE")
diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp
index db5ad31980..7bbc383484 100644
--- a/src/conditioning/conditioner.hpp
+++ b/src/conditioning/conditioner.hpp
@@ -16,6 +16,7 @@
 #include "model/te/clip.hpp"
 #include "model/te/llada_image_te.hpp"
 #include "model/te/llm.hpp"
+#include "model/te/ming_image_te.hpp"
 #include "model/te/t5.hpp"
 #include "model_loader.h"
 #include "tokenizers/sensenova_u1_tokenizer.h"
@@ -398,7 +399,7 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str());
         }
 
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
@@ -816,7 +817,7 @@ struct SD3CLIPEmbedder : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str());
         }
 
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
@@ -908,6 +909,31 @@ struct SD3CLIPEmbedder : public Conditioner {
 
         size_t chunk_count = std::max(std::max(clip_l_tokens.size(), clip_g_tokens.size()), t5_tokens.size()) / chunk_len;
 
+        // Each active encoder must cover every chunk of the longest token sequence.
+        Tokenizer* tokenizers[] = {
+            clip_l ? clip_l_tokenizer.get() : nullptr,
+            clip_g ? clip_g_tokenizer.get() : nullptr,
+            t5 ? &t5_tokenizer : nullptr,
+        };
+        const size_t target_length = chunk_count * chunk_len;
+
+        for (size_t encoder_idx = 0; encoder_idx < 3; encoder_idx++) {
+            auto& tokens  = token_and_weights[encoder_idx].first;
+            auto& weights = token_and_weights[encoder_idx].second;
+            if (!tokenizers[encoder_idx] || tokens.size() >= target_length) {
+                continue;
+            }
+
+            std::vector<int> empty_tokens;
+            std::vector<float> empty_weights;
+            tokenizers[encoder_idx]->pad_tokens(empty_tokens, &empty_weights, nullptr, chunk_len, chunk_len, false);
+
+            while (tokens.size() < target_length) {
+                tokens.insert(tokens.end(), empty_tokens.begin(), empty_tokens.end());
+                weights.insert(weights.end(), empty_weights.begin(), empty_weights.end());
+            }
+        }
+
         for (int chunk_idx = 0; chunk_idx < chunk_count; chunk_idx++) {
             // clip_l
             sd::Tensor<float> chunk_hidden_states_l;
@@ -1206,7 +1232,7 @@ struct FluxCLIPEmbedder : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str());
         }
 
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
@@ -1483,7 +1509,7 @@ struct T5CLIPEmbedder : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str());
         }
 
         auto on_new_token_cb = [&](std::string& str, std::vector<int32_t>& bpe_tokens) -> bool {
@@ -1872,7 +1898,7 @@ struct AnimaConditioner : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str());
         }
 
         std::vector<int> qwen_tokens;
@@ -1969,6 +1995,9 @@ struct LLMEmbedder : public Conditioner {
             if (sd_version_is_pid(version)) {
                 throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer");
             }
+            if (version == VERSION_MOSS_IMAGE) {
+                throw std::runtime_error("MOSS-Image requires the official Qwen3.5 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer");
+            }
         }
         LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL;
         if (version == VERSION_FLUX2) {
@@ -1980,16 +2009,20 @@ struct LLMEmbedder : public Conditioner {
         } else if (sd_version_is_pid(version)) {
             arch = LLM::LLMArch::GEMMA2_2B;
         } else if (version == VERSION_QWEN_IMAGE_2_1 ||
+                   sd_version_is_flux3(version) ||
                    sd_version_is_lingbot_video(version) ||
                    sd_version_is_ideogram4(version) ||
                    sd_version_is_boogu_image(version) ||
                    sd_version_is_sefi_image(version) ||
                    sd_version_is_krea2(version) ||
                    sd_version_is_minimax_h3(version) ||
-                   sd_version_is_mage_flow(version)) {
+                   sd_version_is_mage_flow(version) ||
+                   version == VERSION_IRIS) {
             arch = LLM::LLMArch::QWEN3_VL;
-        } else if (sd_version_is_z_image(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) {
+        } else if (sd_version_is_z_image(version) || sd_version_is_z_image_l2p(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) {
             arch = LLM::LLMArch::QWEN3;
+        } else if (version == VERSION_MOSS_IMAGE) {
+            arch = LLM::LLMArch::QWEN3_5;
         }
         llm        = std::make_shared<LLM::LLMRunner>(arch,
                                                backend,
@@ -2004,6 +2037,8 @@ struct LLMEmbedder : public Conditioner {
             pad_id = 199999;
         } else if (arch == LLM::LLMArch::GEMMA2_2B) {
             pad_id = 0;
+        } else if (arch == LLM::LLMArch::QWEN3_5) {
+            pad_id = 248044;
         }
         tokenizer = tokenizers.create(TokenizerConfig::MAIN, llm->config.vocab_size, pad_id);
         if (!tokenizer) {
@@ -2112,9 +2147,10 @@ struct LLMEmbedder : public Conditioner {
 
     std::tuple<std::vector<int>, std::vector<float>, std::vector<float>> tokenize(std::string text,
                                                                                   const std::pair<int, int>& attn_range,
-                                                                                  size_t min_length = 0,
-                                                                                  size_t max_length = 100000000,
-                                                                                  bool spell_quotes = false) {
+                                                                                  size_t min_length         = 0,
+                                                                                  size_t max_length         = 100000000,
+                                                                                  bool spell_quotes         = false,
+                                                                                  const std::string& suffix = "") {
         std::vector<std::pair<std::string, float>> parsed_attention;
         if (attn_range.first >= 0 && attn_range.second > 0) {
             if (attn_range.first > 0) {
@@ -2143,7 +2179,7 @@ struct LLMEmbedder : public Conditioner {
                 ss << "['" << item.first << "', " << item.second << "], ";
             }
             ss << "]";
-            LOG_VERBOSE("parse '%s' to %s", text.c_str(), ss.str().c_str());
+            LOG_VERBOSE("parse '%s' to %s", escape_newlines(text).c_str(), escape_newlines(ss.str()).c_str());
         }
 
         std::vector<int> tokens;
@@ -2159,6 +2195,23 @@ struct LLMEmbedder : public Conditioner {
             weights.insert(weights.end(), curr_tokens.size(), curr_weight);
         }
 
+        if (!suffix.empty()) {
+            std::vector<int> suffix_tokens;
+            if (!tokenizer->encode(suffix, suffix_tokens) || suffix_tokens.size() >= max_length) {
+                return {};
+            }
+            // Reserve the assistant-turn marker before truncating the caption.
+            if (tokens.size() > max_length - suffix_tokens.size()) {
+                tokens.resize(max_length - suffix_tokens.size());
+                weights.resize(tokens.size());
+            }
+            tokens.insert(tokens.end(), suffix_tokens.begin(), suffix_tokens.end());
+            weights.insert(weights.end(), suffix_tokens.size(), 1.f);
+        }
+
+        if (sd_version_is_flux3(version)) {
+            min_length = std::min(max_length, ((tokens.size() + 79) / 80) * 80);
+        }
         std::vector<float> mask;
         tokenizer->pad_tokens(tokens, &weights, &mask, min_length, max_length);
 
@@ -2181,8 +2234,10 @@ struct LLMEmbedder : public Conditioner {
                                     bool spell_quotes                                       = false,
                                     int max_length                                          = 100000000,
                                     const LLM::DeepStackImageEmbeds& deepstack_image_embeds = {},
-                                    const std::vector<LLM::ImageGrid>& image_grids          = {}) {
-        auto tokens_weights_mask = tokenize(prompt, prompt_attn_range, min_length, max_length, spell_quotes);
+                                    const std::vector<LLM::ImageGrid>& image_grids          = {},
+                                    const std::string& prompt_suffix                        = "",
+                                    sd::Tensor<float>* output_mask                          = nullptr) {
+        auto tokens_weights_mask = tokenize(prompt, prompt_attn_range, min_length, max_length, spell_quotes, prompt_suffix);
         auto& tokens             = std::get<0>(tokens_weights_mask);
         auto& weights            = std::get<1>(tokens_weights_mask);
         auto& mask               = std::get<2>(tokens_weights_mask);
@@ -2210,6 +2265,15 @@ struct LLMEmbedder : public Conditioner {
             }
         }
 
+        sd::Tensor<int32_t> position_ids;
+        if (sd_version_is_flux3(version)) {
+            position_ids = sd::Tensor<int32_t>::zeros({static_cast<int64_t>(tokens.size() * 4)});
+            for (size_t axis = 0; axis < 4; ++axis) {
+                for (size_t i = 0; i < tokens.size(); ++i) {
+                    position_ids[axis * tokens.size() + i] = mask[i] == 0.f ? 1 : static_cast<int32_t>(i);
+                }
+            }
+        }
         auto hidden_states = llm->compute(n_threads,
                                           input_ids,
                                           attention_mask,
@@ -2218,8 +2282,12 @@ struct LLMEmbedder : public Conditioner {
                                           false,
                                           false,
                                           deepstack_image_embeds,
-                                          image_grids);
-        GGML_ASSERT(!hidden_states.empty());
+                                          image_grids,
+                                          position_ids);
+        if (hidden_states.empty()) {
+            LOG_ERROR("LLM prompt encoding failed");
+            return {};
+        }
         hidden_states = apply_token_weights(std::move(hidden_states), weights);
         GGML_ASSERT(hidden_states.shape()[1] > prompt_template_encode_start_idx);
 
@@ -2242,6 +2310,12 @@ struct LLMEmbedder : public Conditioner {
                                                 1);
         }
 
+        if (output_mask != nullptr) {
+            *output_mask = sd::Tensor<float>::zeros({new_hidden_states.shape()[1]});
+            for (size_t i = prompt_template_encode_start_idx; i < mask.size(); ++i) {
+                output_mask->data()[i - prompt_template_encode_start_idx] = mask[i];
+            }
+        }
         return new_hidden_states;
     }
 
@@ -2308,6 +2382,8 @@ struct LLMEmbedder : public Conditioner {
         int hidden_states_min_length         = 0;  // zero pad hidden_states
         bool spell_quotes                    = false;
         std::set<int> out_layers;
+        std::string prompt_suffix;
+        sd::Tensor<float> text_mask;
 
         int64_t t0                     = ggml_time_ms();
         RefImageResizeMode resize_mode = conditioner_params.ref_image_params.vlm_resize_mode;
@@ -2949,7 +3025,7 @@ struct LLMEmbedder : public Conditioner {
             prompt_attn_range.second = static_cast<int>(prompt.size());
 
             prompt += "<|end|><|start|>assistant<|channel|>analysis<|message|>Need to generate one image according to the description.<|end|><|start|>assistant<|channel|>final<|message|>";
-        } else if (sd_version_is_z_image(version)) {
+        } else if (sd_version_is_z_image(version) || sd_version_is_z_image_l2p(version)) {
             prompt_template_encode_start_idx = 0;
             out_layers                       = {35};  // -2
 
@@ -2970,6 +3046,20 @@ struct LLMEmbedder : public Conditioner {
 
                 prompt += "<|im_end|>\n<|im_start|>assistant\n";
             }
+        } else if (version == VERSION_MOSS_IMAGE) {
+            prompt_template_encode_start_idx = 0;
+            max_length                       = 4096;
+            prompt                           = "<|im_start|>user\n" + conditioner_params.text + "<|im_end|>\n<|im_start|>assistant\n<think>\n";
+            prompt_attn_range                = {0, 0};
+        } else if (sd_version_is_flux3(version)) {
+            prompt_template_encode_start_idx = 0;
+            max_length                       = 4096;
+            out_layers                       = {4, 8, 12, 16, 20, 24, 28, 32};
+            prompt                           = "<|im_start|>user\n";
+            prompt_attn_range.first          = static_cast<int>(prompt.size());
+            prompt += conditioner_params.text;
+            prompt_attn_range.second = static_cast<int>(prompt.size());
+            prompt += "<|im_end|>\n<|im_start|>assistant\n";
         } else if (version == VERSION_FLUX2_KLEIN) {
             prompt_template_encode_start_idx = 0;
             min_length                       = 512;
@@ -3057,6 +3147,23 @@ struct LLMEmbedder : public Conditioner {
             SDCondition result;
             result.c_crossattn = std::move(hidden_states);
             return result;
+        } else if (version == VERSION_IRIS) {
+            prompt =
+                "<|im_start|>system\n"
+                "Describe the image by detailing the color, shape, size, texture, quantity, text, spatial "
+                "relationships of the objects and background:<|im_end|>\n<|im_start|>user\n";
+            auto prefix_tokens = std::get<0>(tokenize(prompt, {0, 0}));
+            if (prefix_tokens.empty()) {
+                return {};
+            }
+            prompt_template_encode_start_idx = static_cast<int>(prefix_tokens.size());
+            hidden_states_min_length         = 300;
+            max_length                       = prompt_template_encode_start_idx + hidden_states_min_length;
+            out_layers                       = {2, 5, 8, 11, 14, 17, 20, 23, 26, 29, 32, 35};
+            prompt_attn_range.first          = static_cast<int>(prompt.size());
+            prompt += conditioner_params.text;
+            prompt_attn_range.second = static_cast<int>(prompt.size());
+            prompt_suffix            = "<|im_end|>\n<|im_start|>assistant\n";
         } else {
             GGML_ABORT("unknown version %d", version);
         }
@@ -3072,7 +3179,9 @@ struct LLMEmbedder : public Conditioner {
                                            spell_quotes,
                                            max_length,
                                            deepstack_image_embeds,
-                                           image_grids);
+                                           image_grids,
+                                           prompt_suffix,
+                                           version == VERSION_IRIS ? &text_mask : nullptr);
         if (hidden_states.empty()) {
             return {};
         }
@@ -3137,6 +3246,7 @@ struct LLMEmbedder : public Conditioner {
         SDCondition result;
         result.c_crossattn        = std::move(hidden_states);
         result.extra_c_crossattns = std::move(extra_hidden_states_vec);
+        result.c_vector           = std::move(text_mask);
         if (version == VERSION_QWEN_IMAGE_2_1) {
             auto slots = sd::Tensor<int32_t>::zeros({result.c_crossattn.shape()[1]});
             for (size_t i = 0; i < image_embeds.size(); ++i) {
@@ -3163,6 +3273,66 @@ struct LLMEmbedder : public Conditioner {
             int64_t tag_count    = static_cast<int64_t>(tags.size());
             result.c_token_types = sd::Tensor<int32_t>({tag_count}, std::move(tags));
         }
+
+        return result;
+    }
+};
+
+struct MingImageEmbedder : public Conditioner {
+    std::shared_ptr<Tokenizer> tokenizer;
+    std::shared_ptr<MingImageTE::MingImageTextRunner> text_model;
+    const std::string prefix = "text_encoders.llm";
+
+    MingImageEmbedder(ggml_backend_t backend, const String2TensorStorage& tensors, std::shared_ptr<RunnerWeightManager> weight_manager, const TokenizerConfig& tokenizers) {
+        if (!tokenizers.has(TokenizerConfig::MAIN)) {
+            throw std::runtime_error("Ming-Image requires the Ling tokenizer.json; pass --tokenizer FILE");
+        }
+        text_model = std::make_shared<MingImageTE::MingImageTextRunner>(backend, tensors, prefix, weight_manager);
+        tokenizer  = tokenizers.create(TokenizerConfig::MAIN, text_model->config.backbone.vocab_size, 156895);
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        text_model->get_param_tensors(tensors, prefix);
+    }
+    void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& ops) override {
+        text_model->get_param_tensor_ops(ops);
+    }
+    void get_layer_split_param_tensors(std::map<std::string, ggml_tensor*>& tensors) override {
+        text_model->get_param_tensors(tensors, prefix);
+    }
+    void set_flash_attention_enabled(bool enabled) override { text_model->set_flash_attention_enabled(enabled); }
+    void set_max_graph_vram_bytes(size_t bytes) override { text_model->set_max_graph_vram_bytes(bytes); }
+    void set_runtime_backends(const std::vector<ggml_backend_t>& backends) override { text_model->set_runtime_backends(backends); }
+    void set_graph_cut_layer_split_enabled(bool enabled) override { text_model->set_graph_cut_layer_split_enabled(enabled); }
+    void set_graph_cut_layer_split_backend_vram_limits(const std::vector<size_t>& limits) override { text_model->set_graph_cut_layer_split_backend_vram_limits(limits); }
+    void set_scale_overrides(float linear, float attention) override { text_model->set_scale_overrides(linear, attention); }
+    void set_weight_adapter(const std::shared_ptr<WeightAdapter>& adapter) override { text_model->set_weight_adapter(adapter); }
+    void runner_end() override { text_model->runner_end(); }
+
+    SDCondition get_learned_condition(int n_threads, const ConditionerParams& input) override {
+        if (input.ref_images != nullptr && !input.ref_images->empty()) {
+            LOG_ERROR("Ming-Image currently supports text-to-image only");
+            return {};
+        }
+        std::string prompt =
+            "<role>SYSTEM</role>你是一个友好的AI助手。\n\ndetailed thinking off<|role_end|>"
+            "<role>HUMAN</role>" +
+            input.text + "<|role_end|><role>ASSISTANT</role>";
+        std::vector<int> tokens;
+        if (!tokenizer->encode(prompt, tokens, nullptr)) {
+            return {};
+        }
+        if (tokens.size() + 258 > 32768) {
+            LOG_ERROR("Ming-Image prompt exceeds the text encoder context length");
+            return {};
+        }
+        auto output = text_model->compute(n_threads, tokens);
+        if (output.empty()) {
+            return {};
+        }
+        SDCondition result;
+        result.c_crossattn = sd::ops::slice(sd::ops::slice(output, 1, 0, 256), 0, 0, text_model->config.caption_dim);
+        result.extra_c_crossattns.push_back(sd::ops::slice(output, 1, 256, output.shape()[1]));
         return result;
     }
 };
diff --git a/src/conditioning/conditioning_cache.h b/src/conditioning/conditioning_cache.h
new file mode 100644
index 0000000000..6e6cbcaf77
--- /dev/null
+++ b/src/conditioning/conditioning_cache.h
@@ -0,0 +1,107 @@
+#ifndef __SD_CONDITIONING_CONDITIONING_CACHE_H__
+#define __SD_CONDITIONING_CONDITIONING_CACHE_H__
+
+#include <algorithm>
+#include <list>
+#include <tuple>
+
+#include "conditioning/conditioner.hpp"
+
+class ConditioningCache {
+    struct Entry {
+        ConditionerParams params;
+        std::vector<sd::Tensor<float>> ref_images;
+        std::vector<MiniMaxH3PresentationItem> references;
+        SDCondition condition;
+
+        Entry(const ConditionerParams& input, const SDCondition& output)
+            : params(input), condition(output) {
+            // Request-owned reference pointers must not outlive the request.
+            if (input.ref_images != nullptr) {
+                ref_images        = *input.ref_images;
+                params.ref_images = &ref_images;
+            }
+            if (input.minimax_h3_references != nullptr) {
+                references                   = *input.minimax_h3_references;
+                params.minimax_h3_references = &references;
+            }
+        }
+
+        Entry(const Entry&)            = delete;
+        Entry& operator=(const Entry&) = delete;
+    };
+
+    size_t capacity_ = 4;
+    std::list<Entry> entries_;
+
+    static bool same_images(const std::vector<sd::Tensor<float>>& a,
+                            const std::vector<sd::Tensor<float>>& b) {
+        return std::equal(a.begin(), a.end(), b.begin(), b.end(),
+                          [](const sd::Tensor<float>& x, const sd::Tensor<float>& y) {
+                              return x.shape() == y.shape() && x.values() == y.values();
+                          });
+    }
+
+    static bool same_params(const ConditionerParams& a, const ConditionerParams& b) {
+        const auto fields = [](const ConditionerParams& p) {
+            const auto& r = p.ref_image_params;
+            return std::tie(p.text, p.clip_skip, p.width, p.height, p.zero_out_masked,
+                            r.pass_to_vlm, r.pass_to_dit, r.ref_index_mode,
+                            r.force_ref_timestep_zero, r.resize_before_vae, r.vae_input_max_pixels,
+                            r.vlm_resize_mode, r.vlm_min_size, r.vlm_max_size, r.resize_vae_to_target);
+        };
+        if (fields(a) != fields(b) ||
+            (a.ref_images == nullptr) != (b.ref_images == nullptr) ||
+            (a.minimax_h3_references == nullptr) != (b.minimax_h3_references == nullptr)) {
+            return false;
+        }
+        if (a.ref_images != nullptr && !same_images(*a.ref_images, *b.ref_images)) {
+            return false;
+        }
+        if (a.minimax_h3_references != nullptr &&
+            !std::equal(a.minimax_h3_references->begin(), a.minimax_h3_references->end(),
+                        b.minimax_h3_references->begin(), b.minimax_h3_references->end(),
+                        [](const MiniMaxH3PresentationItem& x, const MiniMaxH3PresentationItem& y) {
+                            return x.kind == y.kind && x.timestamps == y.timestamps && same_images(x.frames, y.frames);
+                        })) {
+            return false;
+        }
+        return true;
+    }
+
+public:
+    void set_capacity(size_t capacity) {
+        capacity_ = capacity;
+        while (entries_.size() > capacity_) {
+            entries_.pop_back();
+        }
+    }
+
+    void clear() {
+        entries_.clear();
+    }
+
+    SDCondition get(Conditioner& conditioner, int n_threads, const ConditionerParams& params) {
+        if (capacity_ == 0) {
+            return conditioner.get_learned_condition(n_threads, params);
+        }
+        for (auto it = entries_.begin(); it != entries_.end(); ++it) {
+            if (same_params(it->params, params)) {
+                entries_.splice(entries_.begin(), entries_, it);
+                LOG_INFO("conditioning cache hit");
+                return entries_.front().condition;
+            }
+        }
+        auto condition = conditioner.get_learned_condition(n_threads, params);
+        if (!condition.empty()) {
+            if (entries_.size() == capacity_) {
+                entries_.pop_back();
+            }
+            entries_.emplace_front(params, condition);
+            LOG_VERBOSE("conditioning cache stored (%zu/%zu)", entries_.size(), capacity_);
+        }
+        return condition;
+    }
+};
+
+#endif  // __SD_CONDITIONING_CONDITIONING_CACHE_H__
diff --git a/src/convert.cpp b/src/convert.cpp
index c82d2557ac..b8a62121d5 100644
--- a/src/convert.cpp
+++ b/src/convert.cpp
@@ -3,7 +3,9 @@
 #include <cstdint>
 #include <cstring>
 #include <exception>
+#include <filesystem>
 #include <fstream>
+#include <map>
 #include <memory>
 #include <mutex>
 #include <regex>
@@ -11,11 +13,15 @@
 #include <thread>
 #include <vector>
 
+#include "core/ggml_extend_backend.h"
 #include "core/util.h"
 #include "model_io/gguf_io.h"
 #include "model_io/safetensors_io.h"
 #include "model_io/streaming_writer.h"
 #include "model_loader.h"
+#include "model_manager.h"
+#include "name_conversion.h"
+#include "runtime/imatrix.h"
 
 struct TensorExportInfo {
     TensorStorage storage;
@@ -75,9 +81,9 @@ static size_t export_tensor_nbytes(const TensorExportInfo& info) {
     return static_cast<size_t>(output_storage.nbytes());
 }
 
-static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo& info) {
+static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo& info, const std::string& name) {
     TensorWritePlan plan;
-    plan.name   = info.storage.name;
+    plan.name   = name;
     plan.type   = info.type;
     plan.n_dims = info.storage.n_dims;
     for (int i = 0; i < SD_MAX_DIMS; i++) {
@@ -86,11 +92,14 @@ static TensorWritePlan tensor_write_plan_from_export_info(const TensorExportInfo
     return plan;
 }
 
-static std::vector<TensorWritePlan> tensor_write_plans_from_export_infos(const std::vector<TensorExportInfo>& tensors) {
+static std::vector<TensorWritePlan> tensor_write_plans_from_export_infos(const std::vector<TensorExportInfo>& tensors,
+                                                                         bool convert_name,
+                                                                         SDVersion version) {
     std::vector<TensorWritePlan> plans;
     plans.reserve(tensors.size());
     for (const TensorExportInfo& info : tensors) {
-        plans.push_back(tensor_write_plan_from_export_info(info));
+        const std::string name = convert_name ? convert_tensor_name(info.storage.name, version) : info.storage.name;
+        plans.push_back(tensor_write_plan_from_export_info(info, name));
     }
     return plans;
 }
@@ -100,7 +109,7 @@ static bool preallocate_output_file(const std::string& output_path, uint64_t fil
         return true;
     }
 
-    std::fstream file(output_path, std::ios::binary | std::ios::in | std::ios::out);
+    std::fstream file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out);
     if (!file.is_open()) {
         if (error != nullptr) {
             *error = "failed to open output file '" + output_path + "' for preallocation";
@@ -158,16 +167,42 @@ static bool load_tensor_for_export(ModelLoader& model_loader, TensorExportJob& j
     return true;
 }
 
+static bool export_tensor_from_memory(const TensorExportInfo& info,
+                                      const std::map<std::string, ggml_tensor*>& mem,
+                                      SDVersion version,
+                                      std::vector<uint8_t>& out) {
+    auto found = mem.find(info.storage.name);
+    if (found == mem.end() || found->second == nullptr || found->second->data == nullptr) {
+        return false;
+    }
+    const TensorStorage& storage = info.storage;
+    out.resize(export_tensor_nbytes(info));
+    if (out.size() > 0) {
+        std::vector<float> imatrix = get_imatrix_collector().get_values(convert_tensor_name(info.storage.name, version));
+        convert_tensor(found->second->data,
+                       found->second->type,
+                       out.data(),
+                       info.type,
+                       (int)(storage.nelements() / storage.ne[0]),
+                       (int)storage.ne[0],
+                       std::move(imatrix));
+    }
+    return true;
+}
+
 static bool stream_tensor_data(ModelLoader& model_loader,
                                const std::string& output_path,
                                const std::vector<TensorExportInfo>& tensors,
                                const StreamingModelWriter& writer,
                                int n_threads,
+                               const std::map<std::string, ggml_tensor*>* mem,
                                std::string* error) {
     n_threads = n_threads > 0 ? n_threads : sd_get_num_physical_cores();
     n_threads = std::max(1, n_threads);
     LOG_INFO("streaming convert with %d threads", n_threads);
 
+    const SDVersion version = model_loader.get_sd_version();
+
     int64_t start_time       = ggml_time_ms();
     uint64_t bytes_written   = 0;
     size_t tensors_written   = 0;
@@ -217,7 +252,7 @@ static bool stream_tensor_data(ModelLoader& model_loader,
 
     for (int worker = 0; worker < n_threads; worker++) {
         workers.emplace_back([&]() {
-            std::fstream output_file(output_path, std::ios::binary | std::ios::in | std::ios::out);
+            std::fstream output_file(std::filesystem::u8path(output_path), std::ios::binary | std::ios::in | std::ios::out);
             if (!output_file.is_open()) {
                 fail("failed to open output file '" + output_path + "' for tensor writing");
                 return;
@@ -241,7 +276,10 @@ static bool stream_tensor_data(ModelLoader& model_loader,
                 TensorExportJob job;
                 job.info = tensors[tensor_index];
                 try {
-                    job.success = load_tensor_for_export(model_loader, job);
+                    job.success = (mem != nullptr) && export_tensor_from_memory(job.info, *mem, version, job.data);
+                    if (!job.success) {
+                        job.success = load_tensor_for_export(model_loader, job);
+                    }
                 } catch (const std::exception& e) {
                     job.error   = e.what();
                     job.success = false;
@@ -298,8 +336,10 @@ static bool write_model_file_streaming(ModelLoader& model_loader,
                                        const std::vector<TensorExportInfo>& tensors,
                                        StreamingModelWriter& writer,
                                        int n_threads,
+                                       bool convert_name,
+                                       const std::map<std::string, ggml_tensor*>* mem,
                                        std::string* error) {
-    std::vector<TensorWritePlan> plans = tensor_write_plans_from_export_infos(tensors);
+    std::vector<TensorWritePlan> plans = tensor_write_plans_from_export_infos(tensors, convert_name, model_loader.get_sd_version());
     if (!writer.write_metadata(output_path, plans, error)) {
         return false;
     }
@@ -307,7 +347,7 @@ static bool write_model_file_streaming(ModelLoader& model_loader,
         return false;
     }
     model_loader.process_model_files(false, false);
-    return stream_tensor_data(model_loader, output_path, tensors, writer, n_threads, error);
+    return stream_tensor_data(model_loader, output_path, tensors, writer, n_threads, mem, error);
 }
 
 static bool init_convert_path(ModelLoader& model_loader, const char* path, const char* prefix, bool& loaded_any) {
@@ -326,7 +366,9 @@ static bool export_loaded_model(ModelLoader& model_loader,
                                 const char* output_path,
                                 sd_type_t output_type,
                                 const char* tensor_type_rules,
-                                int n_threads) {
+                                int n_threads,
+                                bool convert_name,
+                                const std::map<std::string, ggml_tensor*>* mem = nullptr) {
     ggml_type type             = sd_type_to_ggml_type(output_type);
     bool output_is_safetensors = ends_with(output_path, ".safetensors");
     TensorTypeRules type_rules = parse_tensor_type_rules(tensor_type_rules);
@@ -341,7 +383,7 @@ static bool export_loaded_model(ModelLoader& model_loader,
         } else {
             writer = std::make_unique<GGUFStreamingWriter>();
         }
-        success = write_model_file_streaming(model_loader, output_path, tensors, *writer, n_threads, &error);
+        success = write_model_file_streaming(model_loader, output_path, tensors, *writer, n_threads, convert_name, mem, &error);
     }
 
     if (!success && !error.empty()) {
@@ -351,6 +393,174 @@ static bool export_loaded_model(ModelLoader& model_loader,
     return success;
 }
 
+static bool has_active_loras(const sd_lora_t* loras, int lora_count) {
+    if (loras == nullptr) {
+        return false;
+    }
+    for (int i = 0; i < lora_count; i++) {
+        if (loras[i].multiplier != 0.0f) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static std::vector<ModelManager::LoraSpec> build_lora_specs(const sd_lora_t* loras, int lora_count) {
+    std::vector<ModelManager::LoraSpec> specs;
+    specs.reserve(lora_count);
+    for (int i = 0; i < lora_count; i++) {
+        ModelManager::LoraSpec spec;
+        spec.path          = loras[i].path != nullptr ? loras[i].path : "";
+        spec.multiplier    = loras[i].multiplier;
+        spec.is_high_noise = loras[i].is_high_noise;
+        spec.required      = true;
+        specs.push_back(std::move(spec));
+    }
+    return specs;
+}
+
+static ModelComponent component_for_tensor_name(const std::string& name) {
+    if (starts_with(name, "text_encoders.") || starts_with(name, "cond_stage_model.")) {
+        return ModelComponent::Conditioner;
+    }
+    if (starts_with(name, "model.high_noise_diffusion_model.")) {
+        return ModelComponent::HighNoiseDiffusion;
+    }
+    if (starts_with(name, "model.diffusion_model.")) {
+        return ModelComponent::Diffusion;
+    }
+    if (starts_with(name, "vae.") || starts_with(name, "first_stage_model.")) {
+        return ModelComponent::VAE;
+    }
+    return ModelComponent::Diffusion;
+}
+
+static bool load_model_into_memory(ModelLoader& model_loader,
+                                   const sd_lora_t* loras,
+                                   int lora_count,
+                                   int n_threads,
+                                   ModelManager& manager,
+                                   ggml_backend_t cpu,
+                                   ggml_context* ctx,
+                                   std::map<std::string, ggml_tensor*>& mem,
+                                   std::vector<ggml_tensor*>& all_tensors,
+                                   ggml_type output_type,
+                                   const TensorTypeRules& type_rules) {
+    const String2TensorStorage& storage_map = model_loader.get_tensor_storage_map();
+    if (storage_map.empty()) {
+        LOG_ERROR("no tensors to load into memory for convert");
+        return false;
+    }
+
+    const SDVersion version  = model_loader.get_sd_version();
+    ModelLoader merge_loader = model_loader;
+    merge_loader.convert_tensors_name();
+
+    std::map<ModelComponent, std::map<std::string, ggml_tensor*>> groups;
+    size_t total_bytes = 0;
+    for (const auto& [name, storage] : storage_map) {
+        ggml_type load_type         = storage.type;
+        const ggml_type export_type = get_export_tensor_type(model_loader, storage, output_type, type_rules);
+        const bool quant_src        = ggml_is_quantized(storage.type);
+        const bool quant_dst        = ggml_is_quantized(export_type);
+        if (quant_src && quant_dst) {
+            load_type = GGML_TYPE_F16;
+        } else if (quant_src || quant_dst) {
+            load_type = quant_src ? export_type : storage.type;
+        } else if (storage.type == GGML_TYPE_F32 || export_type == GGML_TYPE_F32) {
+            load_type = GGML_TYPE_F32;
+        }
+        ggml_tensor* tensor = ggml_new_tensor(ctx, load_type, storage.n_dims, storage.ne);
+        if (tensor == nullptr) {
+            LOG_ERROR("failed to create tensor '%s' for in-memory convert", name.c_str());
+            return false;
+        }
+        const std::string merge_name = convert_tensor_name(name, version);
+        auto& group                  = groups[component_for_tensor_name(merge_name)];
+        if (!group.emplace(merge_name, tensor).second) {
+            LOG_ERROR("duplicate tensor name '%s' after name conversion", merge_name.c_str());
+            return false;
+        }
+        mem[name] = tensor;
+        all_tensors.push_back(tensor);
+        total_bytes += ggml_nbytes(tensor);
+    }
+    LOG_INFO("loading %zu tensors (%.2f GB) into memory for convert",
+             all_tensors.size(), total_bytes / (1024.0 * 1024.0 * 1024.0));
+
+    manager.set_n_threads(n_threads);
+    manager.set_enable_mmap(false);
+    if (!manager.set_loader(std::move(merge_loader))) {
+        LOG_ERROR("failed to set model loader for in-memory convert");
+        return false;
+    }
+
+    for (const auto& [component, tensors] : groups) {
+        if (!manager.register_param_tensors(component, tensors, ModelManager::ResidencyMode::ParamBackend, cpu, cpu)) {
+            LOG_ERROR("failed to register %s tensors for in-memory convert", model_component_name(component));
+            return false;
+        }
+    }
+
+    std::vector<ModelManager::LoraSpec> lora_specs = build_lora_specs(loras, lora_count);
+    if (!manager.prepare_lora_sources(lora_specs)) {
+        LOG_ERROR("failed to prepare LoRA sources for convert");
+        return false;
+    }
+    if (!manager.set_loras(lora_specs, version)) {
+        LOG_ERROR("failed to set LoRAs for convert");
+        return false;
+    }
+    if (!manager.prepare_params(all_tensors)) {
+        LOG_ERROR("failed to load model into memory for convert");
+        return false;
+    }
+
+    return true;
+}
+
+static bool convert_model_in_memory(ModelLoader& model_loader,
+                                    const char* output_path,
+                                    sd_type_t output_type,
+                                    const char* tensor_type_rules,
+                                    bool convert_name,
+                                    int n_threads,
+                                    const sd_lora_t* loras,
+                                    int lora_count) {
+    auto cpu = std::unique_ptr<ggml_backend, decltype(&ggml_backend_free)>(sd_backend_cpu_init(), ggml_backend_free);
+    if (cpu == nullptr) {
+        LOG_ERROR("failed to init CPU backend for in-memory convert");
+        return false;
+    }
+
+    ggml_init_params ctx_params;
+    ctx_params.mem_size   = model_loader.get_tensor_storage_map().size() * ggml_tensor_overhead();
+    ctx_params.mem_buffer = nullptr;
+    ctx_params.no_alloc   = true;
+    ggml_context* ctx     = ggml_init(ctx_params);
+    if (ctx == nullptr) {
+        LOG_ERROR("ggml_init failed for in-memory convert");
+        return false;
+    }
+
+    ggml_type type             = sd_type_to_ggml_type(output_type);
+    TensorTypeRules type_rules = parse_tensor_type_rules(tensor_type_rules);
+
+    std::vector<ggml_tensor*> all_tensors;
+    std::map<std::string, ggml_tensor*> mem;
+    bool success = false;
+    {
+        // The manager owns the buffers backing `mem` and must outlive the export; it is
+        // destroyed (freeing those buffers) before the tensor context is freed below.
+        ModelManager manager;
+        if (load_model_into_memory(model_loader, loras, lora_count, n_threads, manager, cpu.get(), ctx, mem, all_tensors, type, type_rules)) {
+            success = export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads, convert_name, &mem);
+        }
+    }
+    ggml_free(ctx);
+    return success;
+}
+
 bool convert_with_components(const char* model_path,
                              const char* clip_l_path,
                              const char* clip_g_path,
@@ -361,10 +571,22 @@ bool convert_with_components(const char* model_path,
                              sd_type_t output_type,
                              const char* tensor_type_rules,
                              bool convert_name,
-                             int n_threads) {
+                             int n_threads,
+                             const sd_lora_t* loras,
+                             int lora_count) {
     if (!validate_tensor_types(output_type, tensor_type_rules)) {
         return false;
     }
+
+    if (loras != nullptr) {
+        for (int i = 0; i < lora_count; i++) {
+            LOG_INFO("lora %d: '%s' (multiplier %.2f%s)",
+                     i + 1, loras[i].path != nullptr ? loras[i].path : "",
+                     loras[i].multiplier,
+                     loras[i].is_high_noise ? ", high noise" : "");
+        }
+    }
+
     ModelLoader model_loader;
     bool loaded_any = false;
 
@@ -382,11 +604,17 @@ bool convert_with_components(const char* model_path,
         return false;
     }
 
+    if (has_active_loras(loras, lora_count)) {
+        LOG_INFO("loading the full model into memory to apply LoRAs");
+        return convert_model_in_memory(model_loader, output_path, output_type, tensor_type_rules, convert_name, n_threads,
+                                       loras, lora_count);
+    }
+
     if (convert_name) {
         model_loader.convert_tensors_name();
     }
 
-    return export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads);
+    return export_loaded_model(model_loader, output_path, output_type, tensor_type_rules, n_threads, false, nullptr);
 }
 
 bool convert(const char* input_path,
@@ -405,5 +633,7 @@ bool convert(const char* input_path,
                                    output_type,
                                    tensor_type_rules,
                                    convert_name,
+                                   0,
+                                   nullptr,
                                    0);
 }
diff --git a/src/core/backend_fit.cpp b/src/core/backend_fit.cpp
index 85cfc727c3..38453def1c 100644
--- a/src/core/backend_fit.cpp
+++ b/src/core/backend_fit.cpp
@@ -478,27 +478,58 @@ namespace sd::backend_fit {
         return true;
     }
 
-    bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params, bool prefer_temporal_tiling) {
+    bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params,
+                                         bool prefer_temporal_tiling,
+                                         ggml_status status,
+                                         int latent_tile_size_w,
+                                         int latent_tile_size_h,
+                                         int scale_factor) {
+        return prepare_vae_retry_tiling(tiling_params, prefer_temporal_tiling, status,
+                                        latent_tile_size_w, latent_tile_size_h, scale_factor, false);
+    }
+
+    bool prepare_vae_retry_tiling(sd_tiling_params_t& tiling_params,
+                                  bool prefer_temporal_tiling,
+                                  ggml_status status,
+                                  int latent_tile_size_w,
+                                  int latent_tile_size_h,
+                                  int scale_factor,
+                                  bool encode) {
+        // Execution failures can leave the device unusable; tiling only helps with allocation failures.
+        if (status != GGML_STATUS_ALLOC_FAILED) {
+            return false;
+        }
+        const char* stage      = encode ? "encode" : "decode";
         const char* retry_mode = nullptr;
         if (prefer_temporal_tiling && !tiling_params.temporal_tiling) {
             tiling_params.temporal_tiling = true;
             retry_mode                    = tiling_params.enabled ? "spatial+temporal" : "temporal";
-        } else if (!tiling_params.enabled) {
-            tiling_params.enabled    = true;
-            tiling_params.rel_size_x = 0.5f;
-            tiling_params.rel_size_y = 0.5f;
-            if (tiling_params.tile_size_x <= 0) {
-                tiling_params.tile_size_x = 256;
+        } else {
+            if (latent_tile_size_w <= 0 || latent_tile_size_h <= 0 || scale_factor <= 0) {
+                return false;
             }
-            if (tiling_params.tile_size_y <= 0) {
-                tiling_params.tile_size_y = 256;
+            auto smaller_tile = [&](int size) {
+                return std::min(size, std::max(4, size / 2));
+            };
+            const int tile_size_w = smaller_tile(latent_tile_size_w);
+            const int tile_size_h = smaller_tile(latent_tile_size_h);
+            if (tile_size_w == latent_tile_size_w && tile_size_h == latent_tile_size_h) {
+                return false;
             }
-            retry_mode = tiling_params.temporal_tiling ? "spatial+temporal" : "spatial";
-        } else {
-            return false;
+            tiling_params.enabled     = true;
+            tiling_params.rel_size_w  = 0.0f;
+            tiling_params.rel_size_h  = 0.0f;
+            tiling_params.tile_size_w = tile_size_w * scale_factor;
+            tiling_params.tile_size_h = tile_size_h * scale_factor;
+            retry_mode                = tiling_params.temporal_tiling ? "spatial+temporal" : "spatial";
+            LOG_WARN("Reducing VAE %s tiles from %dx%d to %dx%d image pixels",
+                     stage,
+                     latent_tile_size_w * scale_factor, latent_tile_size_h * scale_factor,
+                     tiling_params.tile_size_w, tiling_params.tile_size_h);
         }
 
-        LOG_WARN("VAE decode failed (likely out of memory); retrying with %s tiling",
+        LOG_WARN("VAE %s ran out of memory; retrying with %s tiling",
+                 stage,
                  retry_mode);
         return true;
     }
diff --git a/src/core/backend_fit.h b/src/core/backend_fit.h
index 9ef298b3b4..40b5bbdf52 100644
--- a/src/core/backend_fit.h
+++ b/src/core/backend_fit.h
@@ -16,7 +16,20 @@ namespace sd::backend_fit {
                               std::string& params_spec);
 
     bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params,
-                                         bool prefer_temporal_tiling);
+                                         bool prefer_temporal_tiling,
+                                         ggml_status status,
+                                         int latent_tile_size_w,
+                                         int latent_tile_size_h,
+                                         int scale_factor);
+
+    // Same as prepare_vae_decode_retry_tiling, for an encode when encode is true (only the log differs).
+    bool prepare_vae_retry_tiling(sd_tiling_params_t& tiling_params,
+                                  bool prefer_temporal_tiling,
+                                  ggml_status status,
+                                  int latent_tile_size_w,
+                                  int latent_tile_size_h,
+                                  int scale_factor,
+                                  bool encode);
 
 }  // namespace sd::backend_fit
 
diff --git a/src/core/compute_workspace.cpp b/src/core/compute_workspace.cpp
index 54f0538f07..53d80e2f05 100644
--- a/src/core/compute_workspace.cpp
+++ b/src/core/compute_workspace.cpp
@@ -1,6 +1,7 @@
 #include "core/compute_workspace.h"
 
 #include <algorithm>
+#include <atomic>
 #include <cstring>
 #include <exception>
 #include <map>
@@ -14,10 +15,24 @@
 #include "ggml-impl.h"
 
 namespace sd {
+    ComputeWorkspace::ComputeWorkspace(ggml_backend_t backend)
+        : backend_(backend) {
+        if (sd_backend_is_cpu(backend_)) {
+            blas_backend_ = sd_backend_blas_init();
+        }
+    }
+
     ComputeWorkspace::~ComputeWorkspace() {
         segment_end();
         release();
         ggml_backend_free(cpu_backend_);
+        ggml_backend_free(blas_backend_);
+    }
+
+    void ComputeWorkspace::set_accelerator_threads(int n_threads) const {
+        if (blas_backend_ != nullptr) {
+            sd_backend_set_n_threads_any(blas_backend_, n_threads);
+        }
     }
 
     void ComputeWorkspace::set_extra_backends(const std::vector<ggml_backend_t>& backends) {
@@ -33,7 +48,11 @@ namespace sd {
             return true;
         }
         for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
-            if (!ggml_backend_supports_op(backend_, ggml_graph_node(graph, i))) {
+            ggml_tensor* node = ggml_graph_node(graph, i);
+            if (!ggml_backend_supports_op(backend_, node)) {
+                return true;
+            }
+            if (uses_blas() && node->op == GGML_OP_MUL_MAT && ggml_backend_supports_op(blas_backend_, node)) {
                 return true;
             }
         }
@@ -43,6 +62,10 @@ namespace sd {
     ggml_backend_sched_t ComputeWorkspace::make_scheduler(size_t graph_size) {
         std::vector<ggml_backend_t> backends{backend_};
         backends.insert(backends.end(), extra_backends_.begin(), extra_backends_.end());
+        if (uses_blas()) {
+            // Repacked weights are not host buffers, so their matmuls stay on the repack kernels.
+            backends.insert(backends.begin(), blas_backend_);
+        }
         if (!sd_backend_is_cpu(backend_)) {
             if (cpu_backend_ == nullptr) {
                 cpu_backend_ = sd_backend_cpu_init();
@@ -109,6 +132,17 @@ namespace sd {
                 release();
                 return false;
             }
+            if (uses_blas()) {
+                static std::atomic<int> logged{0};
+                if (logged.fetch_add(1) < 4) {
+                    int on_blas = 0;
+                    for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {
+                        on_blas += ggml_backend_sched_get_tensor_backend(scheduler_, ggml_graph_node(graph, i)) == blas_backend_;
+                    }
+                    LOG_INFO("cpu blas: %d of %d graph nodes run on %s", on_blas, ggml_graph_n_nodes(graph),
+                             ggml_backend_name(blas_backend_));
+                }
+            }
         } else {
             if (scheduler_ != nullptr) {
                 release();
@@ -207,11 +241,13 @@ namespace sd {
         auto scheduler   = make_scheduler(graph_size);
         if (scheduler != nullptr) {
             assign_nodes(scheduler, copy_graph);
-            std::vector<size_t> sizes(extra_backends_.size() + 2);
+            std::vector<size_t> sizes(extra_backends_.size() + 3);
             ggml_backend_sched_reserve_size(scheduler, copy_graph, sizes.data());
-            result.buffers.push_back({backend_, sizes[0]});
+            // BLAS shares host memory: count it as the CPU backend's for residency.
+            const size_t first = uses_blas() ? 1 : 0;
+            result.buffers.push_back({backend_, sizes[first] + (first ? sizes[0] : 0)});
             for (size_t i = 0; i < extra_backends_.size(); ++i) {
-                result.buffers.push_back({extra_backends_[i], sizes[i + 1]});
+                result.buffers.push_back({extra_backends_[i], sizes[i + 1 + first]});
             }
             ggml_backend_sched_free(scheduler);
         }
@@ -262,9 +298,22 @@ namespace sd {
         return true;
     }
 
+    void ComputeWorkspace::release_blas_scratch() {
+        if (blas_backend_ == nullptr || active_ || scheduler_ != nullptr) {
+            return;
+        }
+        // ggml-blas keeps its F32 dequantization buffer until the backend is freed.
+        ggml_backend_free(blas_backend_);
+        blas_backend_ = sd_backend_blas_init();
+    }
+
     size_t ComputeWorkspace::bytes(ggml_backend_t backend) const {
         if (scheduler_ != nullptr) {
-            return ggml_backend_sched_get_buffer_size(scheduler_, backend);
+            size_t total = ggml_backend_sched_get_buffer_size(scheduler_, backend);
+            if (backend == backend_ && uses_blas()) {
+                total += ggml_backend_sched_get_buffer_size(scheduler_, blas_backend_);
+            }
+            return total;
         }
         return allocator_ != nullptr && backend == backend_ ? ggml_gallocr_get_buffer_size(allocator_, 0) : 0;
     }
diff --git a/src/core/compute_workspace.h b/src/core/compute_workspace.h
index fcf14b85d1..d1b8cf5bde 100644
--- a/src/core/compute_workspace.h
+++ b/src/core/compute_workspace.h
@@ -17,6 +17,7 @@ namespace sd {
         ggml_backend_t backend_;
         std::vector<ggml_backend_t> extra_backends_;
         ggml_backend_t cpu_backend_     = nullptr;
+        ggml_backend_t blas_backend_    = nullptr;
         ggml_gallocr_t allocator_       = nullptr;
         ggml_backend_sched_t scheduler_ = nullptr;
         size_t scheduler_capacity_      = 0;
@@ -34,8 +35,7 @@ namespace sd {
         };
         using AssignNodes = std::function<void(ggml_backend_sched_t, ggml_cgraph*)>;
 
-        explicit ComputeWorkspace(ggml_backend_t backend)
-            : backend_(backend) {}
+        explicit ComputeWorkspace(ggml_backend_t backend);
         ~ComputeWorkspace();
         ComputeWorkspace(const ComputeWorkspace&)            = delete;
         ComputeWorkspace& operator=(const ComputeWorkspace&) = delete;
@@ -53,9 +53,12 @@ namespace sd {
         void synchronize() const;
         bool segment_end() noexcept;
         bool release();
+        void release_blas_scratch();
+        bool uses_blas() const { return blas_backend_ != nullptr && extra_backends_.empty(); }
         bool active() const { return active_; }
         ggml_backend_sched_t scheduler() const { return scheduler_; }
         ggml_backend_t cpu_backend() const { return cpu_backend_; }
+        void set_accelerator_threads(int n_threads) const;
         size_t bytes(ggml_backend_t backend) const;
         size_t reservation_count() const { return reservations_; }
     };
diff --git a/src/core/ggml_extend.cpp b/src/core/ggml_extend.cpp
index 74324169ee..80e2121c6d 100644
--- a/src/core/ggml_extend.cpp
+++ b/src/core/ggml_extend.cpp
@@ -207,14 +207,28 @@ ggml_tensor* ggml_ext_gelu_quick(ggml_context* ctx,
     return x;
 }
 
-ggml_tensor* ggml_ext_linear(ggml_context* ctx,
-                             ggml_tensor* x,
-                             ggml_tensor* w,
-                             ggml_tensor* b,
-                             bool force_prec_f32,
-                             float scale) {
-    if (scale != 1.f) {
-        x = ggml_ext_scale(ctx, x, scale);
+ggml_tensor* ggml_ext_linear_matmul(ggml_context* ctx,
+                                    ggml_tensor* x,
+                                    ggml_tensor* w,
+                                    bool force_prec_f32) {
+    if (sd_weight_may_be_cpu_repacked(w) && w->ne[2] == 1 && w->ne[3] == 1) {
+        // Repacked CPU matmuls (AMX, CPU_REPACK) get batch dims wrong and only take F32 activations
+        // (the type sd_cpu_repack_buffer_type_for probes with): feed contiguous 2D F32 rows.
+        if (x->type != GGML_TYPE_F32) {
+            x = ggml_cast(ctx, x, GGML_TYPE_F32);
+        }
+        if (!ggml_is_contiguous(x)) {
+            x = ggml_cont(ctx, x);
+        }
+        const int64_t ne1 = x->ne[1];
+        const int64_t ne2 = x->ne[2];
+        const int64_t ne3 = x->ne[3];
+        x                 = ggml_reshape_2d(ctx, x, x->ne[0], ne1 * ne2 * ne3);
+        x                 = ggml_mul_mat(ctx, w, x);
+        if (force_prec_f32) {
+            ggml_prec_set_acc(x, GGML_PREC_F32);
+        }
+        return ggml_reshape_4d(ctx, x, x->ne[0], ne1, ne2, ne3);
     }
     if (x->ne[2] * x->ne[3] > 1024) {
         // workaround: avoid ggml cuda error
@@ -223,15 +237,28 @@ ggml_tensor* ggml_ext_linear(ggml_context* ctx,
         x           = ggml_reshape_2d(ctx, x, x->ne[0], x->ne[1] * x->ne[2] * x->ne[3]);
         x           = ggml_mul_mat(ctx, w, x);
         if (force_prec_f32) {
-            ggml_mul_mat_set_prec(x, GGML_PREC_F32);
+            ggml_prec_set_acc(x, GGML_PREC_F32);
         }
         x = ggml_reshape_4d(ctx, x, x->ne[0], x->ne[1] / ne2 / ne3, ne2, ne3);
     } else {
         x = ggml_mul_mat(ctx, w, x);
         if (force_prec_f32) {
-            ggml_mul_mat_set_prec(x, GGML_PREC_F32);
+            ggml_prec_set_acc(x, GGML_PREC_F32);
         }
     }
+    return x;
+}
+
+ggml_tensor* ggml_ext_linear(ggml_context* ctx,
+                             ggml_tensor* x,
+                             ggml_tensor* w,
+                             ggml_tensor* b,
+                             bool force_prec_f32,
+                             float scale) {
+    if (scale != 1.f) {
+        x = ggml_ext_scale(ctx, x, scale);
+    }
+    x = ggml_ext_linear_matmul(ctx, x, w, force_prec_f32);
     if (scale != 1.f) {
         x = ggml_ext_scale(ctx, x, 1.f / scale);
     }
@@ -255,14 +282,20 @@ ggml_tensor* ggml_ext_linear_i8_tensorwise(ggml_context* ctx,
     }
 
     ggml_tensor* fused_bias = scale == 1.f ? b : nullptr;
+    auto mul_mat            = [&](ggml_tensor* input) {
+        if (input->type == GGML_TYPE_F32 && convrot_group_size > 0) {
+            input = ggml_quantize_i8_convrot(ctx, input, convrot_group_size);
+        }
+        return ggml_mul_mat_i8_tensorwise(ctx, w, input, weight_scale, fused_bias, convrot_group_size);
+    };
     if (x->ne[2] * x->ne[3] > 1024) {
         int64_t ne2 = x->ne[2];
         int64_t ne3 = x->ne[3];
         x           = ggml_reshape_2d(ctx, x, x->ne[0], x->ne[1] * x->ne[2] * x->ne[3]);
-        x           = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size);
+        x           = mul_mat(x);
         x           = ggml_reshape_4d(ctx, x, x->ne[0], x->ne[1] / ne2 / ne3, ne2, ne3);
     } else {
-        x = ggml_mul_mat_i8_tensorwise(ctx, w, x, weight_scale, fused_bias, convrot_group_size);
+        x = mul_mat(x);
     }
 
     if (scale != 1.f) {
@@ -338,22 +371,15 @@ ggml_tensor* ggml_ext_pad(ggml_context* ctx,
 }
 
 static ggml_tensor* conv_1d(ggml_context* ctx, ggml_tensor* x, ggml_tensor* w, int s0, int p0, int d0, bool force_prec_f32) {
-    ggml_tensor* result;
-    if (force_prec_f32) {
-        ggml_tensor* patches = ggml_im2col(ctx, w, x, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F32);
-        result               = ggml_mul_mat(ctx,
-                                            ggml_reshape_2d(ctx, patches, patches->ne[0], patches->ne[2] * patches->ne[1]),
-                                            ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2]));
-        result               = ggml_reshape_3d(ctx, result, patches->ne[1], w->ne[2], patches->ne[2]);
-    } else {
-        result = ggml_conv_1d(ctx, w, x, s0, p0, d0);
+    if (!force_prec_f32) {
+        return ggml_conv_1d(ctx, w, x, s0, p0, d0);
     }
-    if (x->ne[2] > 1) {
-        // mul_mat packs positions and batches before output channels: [OL, N, OC].
-        result = ggml_reshape_3d(ctx, result, result->ne[0], x->ne[2], w->ne[2]);
-        result = ggml_cont(ctx, ggml_permute(ctx, result, 0, 2, 1, 3));
-    }
-    return result;
+    ggml_tensor* patches = ggml_im2col(ctx, w, x, s0, 0, p0, 0, d0, 0, false, GGML_TYPE_F32);
+    ggml_tensor* result  = ggml_mul_mat(ctx,
+                                        ggml_reshape_2d(ctx, patches, patches->ne[0], patches->ne[2] * patches->ne[1]),
+                                        ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1], w->ne[2]));
+    result               = ggml_reshape_3d(ctx, result, patches->ne[1], patches->ne[2], w->ne[2]);
+    return ggml_cont(ctx, ggml_permute(ctx, result, 0, 2, 1, 3));
 }
 
 ggml_tensor* ggml_ext_conv_1d(ggml_context* ctx,
@@ -437,7 +463,14 @@ ggml_tensor* ggml_ext_conv_2d(ggml_context* ctx,
     if (direct) {
         x = ggml_conv_2d_direct(ctx, w, x, s0, s1, p0, p1, d0, d1);
     } else {
-        x = ggml_conv_2d(ctx, w, x, s0, s1, p0, p1, d0, d1);
+        // ggml_conv_2d, but skips the final [OC, N] -> [N, OC] relayout when N == 1 (no data moves).
+        ggml_tensor* im2col = ggml_im2col(ctx, w, x, s0, s1, p0, p1, d0, d1, true,
+                                          w->type == GGML_TYPE_BF16 ? GGML_TYPE_F32 : GGML_TYPE_F16);  // [N, OH, OW, IC * KH * KW]
+        ggml_tensor* out    = ggml_mul_mat(ctx,
+                                           ggml_reshape_2d(ctx, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]),
+                                           ggml_reshape_2d(ctx, w, (w->ne[0] * w->ne[1] * w->ne[2]), w->ne[3]));
+        out                 = ggml_reshape_4d(ctx, out, im2col->ne[1], im2col->ne[2], im2col->ne[3], w->ne[3]);  // [OC, N, OH, OW]
+        x                   = ggml_ext_cont(ctx, ggml_permute(ctx, out, 0, 1, 3, 2));                       // [N, OC, OH, OW]
     }
     if (scale != 1.f) {
         x = ggml_ext_scale(ctx, x, 1.f / scale);
@@ -465,7 +498,11 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
                               int d1,
                               int d2,
                               bool force_prec_f32,
-                              bool direct) {
+                              bool direct,
+                              float scale) {
+    if (scale != 1.f) {
+        x = ggml_ext_scale(ctx, x, scale);
+    }
     if (direct) {
         int64_t OC = w->ne[3] / IC;
         int64_t N  = x->ne[3] / IC;
@@ -478,7 +515,7 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
         x          = ggml_mul_mat(ctx,
                                   ggml_reshape_2d(ctx, im2col, im2col->ne[0], im2col->ne[3] * im2col->ne[2] * im2col->ne[1]),
                                   ggml_reshape_2d(ctx, w, w->ne[0] * w->ne[1] * w->ne[2] * IC, OC));
-        ggml_mul_mat_set_prec(x, GGML_PREC_F32);
+        ggml_prec_set_acc(x, GGML_PREC_F32);
 
         int64_t OD = im2col->ne[3] / N;
         x          = ggml_reshape_4d(ctx, x, im2col->ne[1] * im2col->ne[2], OD, N, OC);
@@ -502,6 +539,9 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
         }
     }
 
+    if (scale != 1.f) {
+        x = ggml_ext_scale(ctx, x, 1.f / scale);
+    }
     if (b != nullptr) {
         b = ggml_reshape_4d(ctx, b, 1, 1, 1, b->ne[0]);  // [OC, 1, 1, 1]
         x = ggml_add_inplace(ctx, x, b);
@@ -623,7 +663,11 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
                                     bool skip_reshape,
                                     bool flash_attn,
                                     float kv_scale,
-                                    bool sage_attn) {  // avoid overflow
+                                    bool sage_attn,
+                                    bool* used_flash_attn) {  // avoid overflow
+    if (used_flash_attn != nullptr) {
+        *used_flash_attn = false;
+    }
     int64_t L_q;
     int64_t L_k;
     int64_t C;
@@ -666,8 +710,8 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
                               v_in->type == GGML_TYPE_F32 && sd_backend_supports_cuda_mma(backend);
         if (pad_head) {
             // CUDA FA MMA starts at 64 channels; keep the original head's attention scale.
-            q_in = ggml_pad(ctx, q_in, 64 - d_head, 0, 0, 0);
-            k_in = ggml_pad(ctx, k_in, 64 - d_head, 0, 0, 0);
+            q_in = ggml_pad(ctx, q_in, static_cast<int>(64 - d_head), 0, 0, 0);
+            k_in = ggml_pad(ctx, k_in, static_cast<int>(64 - d_head), 0, 0, 0);
         }
         if (kv_scale != 1.0f) {
             k_in = ggml_ext_scale(ctx, k_in, kv_scale);
@@ -677,7 +721,7 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
         v_in = ggml_ext_cont(ctx, ggml_permute(ctx, v_in, 0, 2, 1, 3));
         v_in = ggml_reshape_3d(ctx, v_in, d_head, L_k, n_kv_head * N);
         if (pad_head) {
-            v_in = ggml_pad(ctx, v_in, 64 - d_head, 0, 0, 0);
+            v_in = ggml_pad(ctx, v_in, static_cast<int>(64 - d_head), 0, 0, 0);
         }
         if (kv_scale != 1.0f) {
             v_in = ggml_ext_scale(ctx, v_in, kv_scale);
@@ -704,7 +748,7 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
         if (!ggml_backend_supports_op(backend, out)) {
             return nullptr;
         }
-        ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32);
+        ggml_prec_set_acc(out, GGML_PREC_F32);
         if (kv_scale != 1.0f) {
             out = ggml_ext_scale(ctx, out, 1.0f / kv_scale);
         }
@@ -725,9 +769,9 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
         }
         if (padded_head != d_head) {
             // Keep the original head's softmax scale when padding for the CUDA kernel.
-            q_in = ggml_pad(ctx, q_in, padded_head - d_head, 0, 0, 0);
-            k_in = ggml_pad(ctx, k_in, padded_head - d_head, 0, 0, 0);
-            v_in = ggml_pad(ctx, v_in, padded_head - d_head, 0, 0, 0);
+            q_in = ggml_pad(ctx, q_in, static_cast<int>(padded_head - d_head), 0, 0, 0);
+            k_in = ggml_pad(ctx, k_in, static_cast<int>(padded_head - d_head), 0, 0, 0);
+            v_in = ggml_pad(ctx, v_in, static_cast<int>(padded_head - d_head), 0, 0, 0);
         }
         if (kv_scale != 1.0f) {
             k_in = ggml_ext_scale(ctx, k_in, kv_scale);
@@ -744,7 +788,7 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
     }
 #endif
 
-    if (kqv == nullptr && (flash_attn || sage_attn)) {
+    if (kqv == nullptr && flash_attn) {
         // LOG_VERBOSE("attention_ext L_q:%d L_k:%d n_head:%d C:%d d_head:%d N:%d", L_q, L_k, n_head, C, d_head, N);
         bool can_use_flash_attn = true;
         if (mask != nullptr) {
@@ -755,6 +799,9 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
         if (can_use_flash_attn) {
             kqv = build_kqv(q, k, v, mask);
             if (kqv != nullptr) {
+                if (used_flash_attn != nullptr) {
+                    *used_flash_attn = true;
+                }
                 kqv = ggml_view_4d(ctx,
                                    kqv,
                                    d_head,
@@ -777,7 +824,7 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
         v = ggml_reshape_3d(ctx, v, L_k, d_head, n_kv_head * N);   // [N * n_kv_head, d_head, L_k]
 
         auto kq = ggml_mul_mat(ctx, k, q);  // [N * n_head, L_q, L_k]
-        ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
+        ggml_prec_set_acc(kq, GGML_PREC_F32);
         kq = ggml_scale_inplace(ctx, kq, scale);
         if (mask) {
             kq = ggml_add_inplace(ctx, kq, mask);
@@ -796,6 +843,55 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
     return kqv;
 }
 
+ggml_tensor* ggml_ext_attention_prepared(ggml_context* ctx,
+                                         ggml_backend_t backend,
+                                         ggml_tensor* q,
+                                         ggml_tensor* k,
+                                         ggml_tensor* v,
+                                         int64_t n_head,
+                                         int64_t N,
+                                         float kv_scale,
+                                         ggml_tensor* mask) {
+    GGML_ASSERT(q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F16 && v->type == GGML_TYPE_F16);
+    const int64_t L_q    = q->ne[1];
+    const int64_t d_head = v->ne[0];
+    if (backend == nullptr || d_head < 64 || (mask != nullptr && mask->ne[3] != 1)) {
+        return nullptr;
+    }
+
+    // Same expressions as ggml_ext_attention_ext so the softmax scale rounds identically.
+    float scale = (1.0f / sqrt((float)d_head));
+
+    if (mask != nullptr) {
+        if (mask->ne[1] != L_q) {
+            mask = ggml_repeat(ctx, mask, ggml_new_tensor_4d(ctx, mask->type, mask->ne[0], L_q, mask->ne[2], mask->ne[3]));
+        }
+        mask = ggml_cast(ctx, mask, GGML_TYPE_F16);
+    }
+
+    auto kqv = ggml_flash_attn_ext(ctx, q, k, v, mask, scale / kv_scale, 0, 0);
+    if (!ggml_backend_supports_op(backend, kqv)) {
+        return nullptr;
+    }
+    ggml_prec_set_acc(kqv, GGML_PREC_F32);
+    if (kv_scale != 1.0f) {
+        kqv = ggml_ext_scale(ctx, kqv, 1.0f / kv_scale);
+    }
+    kqv = ggml_view_4d(ctx,
+                       kqv,
+                       d_head,
+                       n_head,
+                       L_q,
+                       N,
+                       kqv->nb[1],
+                       kqv->nb[2],
+                       kqv->nb[1] * n_head,
+                       0);
+    kqv = ggml_ext_cont(ctx, kqv);
+    kqv = ggml_reshape_3d(ctx, kqv, d_head * n_head, L_q, N);  // [N, L_q, C]
+    return kqv;
+}
+
 ggml_tensor* ggml_ext_layer_norm(ggml_context* ctx,
                                  ggml_tensor* x,
                                  ggml_tensor* w,
diff --git a/src/core/ggml_extend.h b/src/core/ggml_extend.h
index 33d8c3bf21..165eb673c9 100644
--- a/src/core/ggml_extend.h
+++ b/src/core/ggml_extend.h
@@ -65,6 +65,12 @@ ggml_tensor* ggml_ext_gelu_quick(ggml_context* ctx,
                                  ggml_tensor* x,
                                  bool inplace = false);
 
+// The matrix multiply of ggml_ext_linear without its input/output scaling and bias.
+ggml_tensor* ggml_ext_linear_matmul(ggml_context* ctx,
+                                    ggml_tensor* x,
+                                    ggml_tensor* w,
+                                    bool force_prec_f32);
+
 ggml_tensor* ggml_ext_linear(ggml_context* ctx,
                              ggml_tensor* x,
                              ggml_tensor* w,
@@ -154,7 +160,8 @@ ggml_tensor* ggml_ext_conv_3d(ggml_context* ctx,
                               int d1              = 1,
                               int d2              = 1,
                               bool force_prec_f32 = false,
-                              bool direct         = false);
+                              bool direct         = false,
+                              float scale         = 1.f);
 
 // w: [OC，IC, KD, 1 * 1]
 // x: [N, IC, ID, IH*IW]
@@ -217,11 +224,29 @@ ggml_tensor* ggml_ext_attention_ext(ggml_context* ctx,
                                     ggml_tensor* k,
                                     ggml_tensor* v,
                                     int64_t n_head,
-                                    ggml_tensor* mask = nullptr,
-                                    bool skip_reshape = false,
-                                    bool flash_attn   = false,
-                                    float kv_scale    = 1.0f,
-                                    bool sage_attn    = false);
+                                    ggml_tensor* mask     = nullptr,
+                                    bool skip_reshape     = false,
+                                    bool flash_attn       = false,
+                                    float kv_scale        = 1.0f,
+                                    bool sage_attn        = false,
+                                    bool* used_flash_attn = nullptr);
+
+// Flash attention on inputs that are already in the head-major layout the kernel reads:
+// q: F32 [d_head, L_q, n_head * N], k/v: F16 [d_head, L_k, n_kv_head * N] with the kv_scale
+// already applied. Builds the same flash_attn_ext + output scaling as ggml_ext_attention_ext
+// (skip_reshape, unmasked), so the result is identical; returns nullptr when the backend lacks
+// flash attention for these inputs (the caller then uses ggml_ext_attention_ext).
+// mask (optional): as for ggml_ext_attention_ext's flash path (mask->ne[3] == 1).
+// return: [N, L_q, n_head * d_head]
+ggml_tensor* ggml_ext_attention_prepared(ggml_context* ctx,
+                                         ggml_backend_t backend,
+                                         ggml_tensor* q,
+                                         ggml_tensor* k,
+                                         ggml_tensor* v,
+                                         int64_t n_head,
+                                         int64_t N,
+                                         float kv_scale,
+                                         ggml_tensor* mask = nullptr);
 
 ggml_tensor* ggml_ext_layer_norm(ggml_context* ctx,
                                  ggml_tensor* x,
diff --git a/src/core/ggml_extend_backend.cpp b/src/core/ggml_extend_backend.cpp
index f3049dc23f..9eafe6f1de 100644
--- a/src/core/ggml_extend_backend.cpp
+++ b/src/core/ggml_extend_backend.cpp
@@ -3,16 +3,24 @@
 #include <algorithm>
 #include <cctype>
 #include <cstdlib>
+#include <map>
 #include <mutex>
 #include <sstream>
 #include <stdexcept>
+#include <tuple>
+#include <utility>
 #include <vector>
 
 #ifdef SD_USE_CUDA
 #include <cuda.h>
 #endif
 
+#ifdef SD_USE_HIP
+#include <hip/hip_runtime_api.h>
+#endif
+
 #include "core/util.h"
+#include "ggml-backend-impl.h"
 #include "ggml-impl.h"
 #include "stable-diffusion.h"
 
@@ -433,23 +441,138 @@ bool sd_backend_is_cpu(ggml_backend_t backend) {
     return dev != nullptr && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU;
 }
 
-bool sd_backend_supports_cuda_mma(ggml_backend_t backend) {
+bool sd_cpu_weight_repack_enabled() {
+    static const bool env_enabled = [] {
+        const char* v = std::getenv("SD_CPU_REPACK");
+        return v == nullptr || !(v[0] == '0' && v[1] == '\0');
+    }();
+    return env_enabled;
+}
+
+// Tag lives in an unused op_params slot of the leaf weight, so a reused address cannot inherit it (unlike a pointer set).
+static constexpr int32_t SD_MATMUL_ONLY_TAG     = 0x53444d4d;  // "SDMM"
+static constexpr size_t SD_MATMUL_ONLY_TAG_SLOT = GGML_MAX_OP_PARAMS / sizeof(int32_t) - 1;
+
+void sd_tag_matmul_only_weight(ggml_tensor* weight) {
+    if (weight != nullptr && weight->op == GGML_OP_NONE) {
+        weight->op_params[SD_MATMUL_ONLY_TAG_SLOT] = SD_MATMUL_ONLY_TAG;
+    }
+}
+
+void sd_untag_matmul_only_weight(ggml_tensor* weight) {
+    if (sd_is_matmul_only_weight(weight)) {
+        weight->op_params[SD_MATMUL_ONLY_TAG_SLOT] = 0;
+    }
+}
+
+bool sd_is_matmul_only_weight(const ggml_tensor* weight) {
+    return weight != nullptr && weight->op == GGML_OP_NONE &&
+           weight->op_params[SD_MATMUL_ONLY_TAG_SLOT] == SD_MATMUL_ONLY_TAG;
+}
+
+bool sd_weight_may_be_cpu_repacked(const ggml_tensor* weight) {
+    if (!sd_is_matmul_only_weight(weight)) {
+        return false;
+    }
+    if (weight->buffer == nullptr) {
+        return sd_cpu_weight_repack_enabled();
+    }
+    // Buffer first: a later context disabling repack must not hide already-repacked weights.
+    ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(weight->buffer);
+    ggml_backend_dev_t dev          = ggml_backend_buft_get_device(buft);
+    return !ggml_backend_buft_is_host(buft) && dev != nullptr && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU;
+}
+
+ggml_backend_buffer_type_t sd_cpu_repack_buffer_type_for(ggml_backend_dev_t cpu_dev, const ggml_tensor* weight) {
+    if (cpu_dev == nullptr || ggml_backend_dev_type(cpu_dev) != GGML_BACKEND_DEVICE_TYPE_CPU ||
+        !sd_is_matmul_only_weight(weight) || !sd_cpu_weight_repack_enabled() ||
+        ggml_n_dims(weight) != 2 || !ggml_is_quantized(weight->type)) {
+        return nullptr;
+    }
+    ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(cpu_dev);
+    auto get_extra_bufts   = reg == nullptr ? nullptr
+                                            : (ggml_backend_dev_get_extra_bufts_t)ggml_backend_reg_get_proc_address(reg, "ggml_backend_dev_get_extra_bufts");
+    if (get_extra_bufts == nullptr) {
+        return nullptr;
+    }
+    static std::mutex cache_mutex;
+    static std::map<std::tuple<ggml_backend_dev_t, int, int64_t, int64_t>, ggml_backend_buffer_type_t> cache;
+    const auto key = std::make_tuple(cpu_dev, (int)weight->type, weight->ne[0], weight->ne[1]);
+    {
+        std::lock_guard<std::mutex> lock(cache_mutex);
+        auto it = cache.find(key);
+        if (it != cache.end()) {
+            return it->second;
+        }
+    }
+    ggml_init_params params;
+    params.mem_size   = ggml_tensor_overhead() * 4;
+    params.mem_buffer = nullptr;
+    params.no_alloc   = true;
+    ggml_context* ctx = ggml_init(params);
+    if (ctx == nullptr) {
+        return nullptr;
+    }
+    // Probe with a real 2D-F32 MUL_MAT per candidate (AMX first), the form ggml_ext_linear_matmul feeds; dummy buffer read only for its type.
+    ggml_tensor* w                    = ggml_new_tensor_2d(ctx, weight->type, weight->ne[0], weight->ne[1]);
+    ggml_tensor* x                    = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, weight->ne[0], 64);
+    ggml_tensor* op                   = ggml_mul_mat(ctx, w, x);
+    ggml_backend_buffer_type_t chosen = nullptr;
+    for (ggml_backend_buffer_type_t* it = get_extra_bufts(cpu_dev); it != nullptr && *it != nullptr; ++it) {
+        ggml_backend_buffer_t probe = ggml_backend_buft_alloc_buffer(*it, 0);
+        if (probe == nullptr) {
+            continue;
+        }
+        w->buffer       = probe;
+        const bool good = ggml_backend_dev_supports_op(cpu_dev, op);
+        w->buffer       = nullptr;
+        ggml_backend_buffer_free(probe);
+        if (good) {
+            chosen = *it;
+            break;
+        }
+    }
+    ggml_free(ctx);
+    std::lock_guard<std::mutex> lock(cache_mutex);
+    cache.emplace(key, chosen);
+    return chosen;
+}
+
+ggml_backend_buffer_t sd_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device,
+                                                          void* ptr,
+                                                          size_t size,
+                                                          size_t max_tensor_size) {
+    ggml_backend_buffer_t buffer = ggml_backend_dev_buffer_from_host_ptr(device, ptr, size, max_tensor_size);
+    if (buffer != nullptr && buffer->context == nullptr) {
+        ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(device);
+        if (reg != nullptr && std::strcmp(ggml_backend_reg_name(reg), "Metal") == 0) {
+            // Metal can wrap a failed mapping in a non-null buffer. Its free callback also
+            // dereferences the missing context, so only release the outer buffer.
+            buffer->iface.free_buffer = nullptr;
+            ggml_backend_buffer_free(buffer);
+            return nullptr;
+        }
+    }
+    return buffer;
+}
+
+static int cuda_compiled_arch(ggml_backend_t backend) {
 #ifdef SD_USE_CUDA
     if (!sd_backend_is(backend, "CUDA")) {
-        return false;
+        return 0;
     }
     auto dev = ggml_backend_get_device(backend);
     if (dev == nullptr) {
-        return false;
+        return 0;
     }
     static std::mutex mutex;
-    static std::unordered_map<ggml_backend_dev_t, bool> cache;
+    static std::unordered_map<ggml_backend_dev_t, int> cache;
     std::lock_guard<std::mutex> lock(mutex);
     auto it = cache.find(dev);
     if (it != cache.end()) {
         return it->second;
     }
-    const bool supported = [&]() {
+    const int arch = [&]() {
         ggml_backend_dev_props props{};
         ggml_backend_dev_get_props(dev, &props);
         CUdevice device;
@@ -458,13 +581,13 @@ bool sd_backend_supports_cuda_mma(ggml_backend_t backend) {
             cuDeviceGetByPCIBusId(&device, props.device_id) != CUDA_SUCCESS ||
             cuDeviceGetAttribute(&major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device) != CUDA_SUCCESS ||
             cuDeviceGetAttribute(&minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device) != CUDA_SUCCESS) {
-            return false;
+            return 0;
         }
         auto reg          = ggml_backend_dev_backend_reg(dev);
         auto get_features = reinterpret_cast<ggml_backend_get_features_t>(
             ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features"));
         if (get_features == nullptr) {
-            return false;
+            return 0;
         }
         // Match ggml's highest compiled architecture for this device, including PTX fallback.
         const int cc      = 100 * major + 10 * minor;
@@ -487,21 +610,110 @@ bool sd_backend_supports_cuda_mma(ggml_backend_t backend) {
                 arch = end;
             }
         }
-        return compiled_arch == 700 || compiled_arch >= 750;
+        return compiled_arch;
     }();
-    cache.emplace(dev, supported);
-    return supported;
+    cache.emplace(dev, arch);
+    return arch;
+#else
+    (void)backend;
+    return 0;
+#endif
+}
+
+bool sd_backend_supports_cuda_mma(ggml_backend_t backend) {
+    const int arch = cuda_compiled_arch(backend);
+    return arch == 700 || arch >= 750;
+}
+
+bool sd_backend_supports_hip_mma(ggml_backend_t backend) {
+#ifdef SD_USE_HIP
+    if (!sd_backend_is(backend, "ROCm")) {
+        return false;
+    }
+    auto dev = ggml_backend_get_device(backend);
+    if (dev == nullptr) {
+        return false;
+    }
+    ggml_backend_dev_props props{};
+    ggml_backend_dev_get_props(dev, &props);
+    int device = 0;
+    hipDeviceProp_t hip_props{};
+    if (props.device_id == nullptr ||
+        hipDeviceGetByPCIBusId(&device, props.device_id) != hipSuccess ||
+        hipGetDeviceProperties(&hip_props, device) != hipSuccess) {
+        return false;
+    }
+    const char* arch = hip_props.gcnArchName;
+    if (std::strncmp(arch, "gfx", 3) != 0) {
+        return false;
+    }
+    char* end        = nullptr;
+    const long value = std::strtol(arch + 3, &end, 16);
+    if (end == arch + 3 || (*end != '\0' && *end != ':')) {
+        return false;
+    }
+    // Match the HIP FA kernels; keep unknown architecture families on the probe path.
+    if (value >= 0x1100 && value < 0x1300) {
+        return true;
+    }
+#ifdef SD_HIP_MFMA
+    return value == 0x908 || value == 0x90a ||
+           value == 0x940 || value == 0x941 || value == 0x942 || value == 0x950;
+#else
+    return false;
+#endif
 #else
     (void)backend;
     return false;
 #endif
 }
 
+bool sd_backend_supports_conv_implicit_gemm(ggml_backend_t backend, bool conv3d) {
+    if (backend == nullptr) {
+        return false;
+    }
+    if (sd_backend_is(backend, "Vulkan") || sd_backend_is(backend, "MUSA")) {
+        return true;
+    }
+    // CUDA/HIP Conv3d also implements SIMT implicit GEMM; Conv2d requires MMA.
+    if (conv3d) {
+        return sd_backend_is(backend, "CUDA") || sd_backend_is(backend, "ROCm");
+    }
+    return cuda_compiled_arch(backend) >= 750 || sd_backend_supports_hip_mma(backend);
+}
+
 ggml_backend_t sd_backend_cpu_init() {
     ggml_backend_load_all_once();
     return ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
 }
 
+ggml_backend_t sd_backend_blas_init() {
+    static const bool enabled = [] {
+        const char* v = std::getenv("SD_CPU_BLAS");
+        return v == nullptr || !(v[0] == '0' && v[1] == '\0');
+    }();
+    if (!enabled) {
+        return nullptr;
+    }
+    ggml_backend_load_all_once();
+    ggml_backend_reg_t reg = ggml_backend_reg_by_name("BLAS");
+    if (reg == nullptr || ggml_backend_reg_dev_count(reg) == 0) {
+        return nullptr;
+    }
+    return ggml_backend_dev_init(ggml_backend_reg_dev_get(reg, 0), nullptr);
+}
+
+bool sd_backend_set_n_threads_any(ggml_backend_t backend, int n_threads) {
+    auto dev = backend == nullptr ? nullptr : ggml_backend_get_device(backend);
+    auto reg = dev == nullptr ? nullptr : ggml_backend_dev_backend_reg(dev);
+    auto fn  = reg == nullptr ? nullptr : (ggml_backend_set_n_threads_t)ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
+    if (fn == nullptr) {
+        return false;
+    }
+    fn(backend, n_threads);
+    return true;
+}
+
 bool sd_backend_cpu_set_n_threads(ggml_backend_t backend, int n_threads) {
     if (backend == nullptr) {
         return false;
@@ -715,6 +927,72 @@ bool sd_parse_backend_assignment(const std::string& spec, SDBackendAssignment* a
     return true;
 }
 
+static bool parse_operation_assignment(const std::string& spec, SDBackendAssignment* assignment, std::string* error, bool convolution) {
+    const std::string operation = convolution ? "direct convolution" : "flash attention";
+    if (assignment == nullptr) {
+        return false;
+    }
+    SDBackendAssignment result;
+    result.set_default("auto");
+    std::istringstream stream(spec);
+    std::string part;
+    while (std::getline(stream, part, ',')) {
+        SDBackendAssignment entry;
+        if (!sd_parse_backend_assignment(part, &entry, error)) {
+            return false;
+        }
+        auto valid_mode = [&](const std::string& mode) {
+            if (mode == "on" || mode == "off" || mode == "auto") {
+                return true;
+            }
+            if (error != nullptr) {
+                *error = "invalid " + operation + " mode '" + mode + "': expected on, off, or auto";
+            }
+            return false;
+        };
+        if (!entry.default_name.empty()) {
+            if (!valid_mode(entry.default_name)) {
+                return false;
+            }
+            result.set_default(entry.default_name);
+        }
+        for (const auto& setting : entry.module_names) {
+            switch (setting.first) {
+                case SDBackendModule::DIFFUSION:
+                case SDBackendModule::VAE:
+                case SDBackendModule::CONTROL_NET:
+                    break;
+                case SDBackendModule::TE:
+                case SDBackendModule::CLIP_VISION:
+                case SDBackendModule::AUDIO_ENCODER:
+                    if (!convolution) {
+                        break;
+                    }
+                    [[fallthrough]];
+                default:
+                    if (error != nullptr) {
+                        *error = operation + " is not configurable for module '" + sd_backend_module_name(setting.first) + "'";
+                    }
+                    return false;
+            }
+            if (!valid_mode(setting.second)) {
+                return false;
+            }
+            result.set_module(setting.first, setting.second);
+        }
+    }
+    *assignment = std::move(result);
+    return true;
+}
+
+bool sd_parse_flash_attention_assignment(const std::string& spec, SDBackendAssignment* assignment, std::string* error) {
+    return parse_operation_assignment(spec, assignment, error, false);
+}
+
+bool sd_parse_conv_direct_assignment(const std::string& spec, SDBackendAssignment* assignment, std::string* error) {
+    return parse_operation_assignment(spec, assignment, error, true);
+}
+
 bool SDBackendAssignment::empty() const {
     return default_name.empty() && module_names.empty();
 }
diff --git a/src/core/ggml_extend_backend.h b/src/core/ggml_extend_backend.h
index b2ebd7eb28..0be0c49314 100644
--- a/src/core/ggml_extend_backend.h
+++ b/src/core/ggml_extend_backend.h
@@ -87,8 +87,25 @@ class SDBackendManager {
 
 bool sd_backend_is(ggml_backend_t backend, const std::string& name);
 bool sd_backend_is_cpu(ggml_backend_t backend);
+
+// CPU weight repacking into ggml extra buffers (MUL_MAT-only, so only weights tagged matmul-only). SD_CPU_REPACK=0 disables.
+bool sd_cpu_weight_repack_enabled();
+void sd_tag_matmul_only_weight(ggml_tensor* weight);
+void sd_untag_matmul_only_weight(ggml_tensor* weight);
+bool sd_is_matmul_only_weight(const ggml_tensor* weight);
+// Unallocated weight: answers whether repacking is enabled.
+bool sd_weight_may_be_cpu_repacked(const ggml_tensor* weight);
+ggml_backend_buffer_type_t sd_cpu_repack_buffer_type_for(ggml_backend_dev_t cpu_dev, const ggml_tensor* weight);
 bool sd_backend_supports_cuda_mma(ggml_backend_t backend);
+bool sd_backend_supports_hip_mma(ggml_backend_t backend);
+bool sd_backend_supports_conv_implicit_gemm(ggml_backend_t backend, bool conv3d);
+ggml_backend_buffer_t sd_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device,
+                                                          void* ptr,
+                                                          size_t size,
+                                                          size_t max_tensor_size);
 ggml_backend_t sd_backend_cpu_init();
+ggml_backend_t sd_backend_blas_init();
+bool sd_backend_set_n_threads_any(ggml_backend_t backend, int n_threads);
 bool sd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads);
 ggml_status sd_backend_graph_compute_with_eval_callback(ggml_backend_t backend,
                                                         ggml_cgraph* gf,
@@ -96,6 +113,8 @@ ggml_status sd_backend_graph_compute_with_eval_callback(ggml_backend_t backend,
                                                         void* callback_eval_user_data);
 std::string sd_backend_resolve_name(const std::string& name);
 bool sd_parse_backend_assignment(const std::string& spec, SDBackendAssignment* assignment, std::string* error);
+bool sd_parse_flash_attention_assignment(const std::string& spec, SDBackendAssignment* assignment, std::string* error);
+bool sd_parse_conv_direct_assignment(const std::string& spec, SDBackendAssignment* assignment, std::string* error);
 const char* sd_backend_module_name(SDBackendModule module);
 void ggml_ext_im_set_f32_1d(const struct ggml_tensor* tensor, int i, float value);
 bool add_rpc_devices(const std::string& servers);
diff --git a/src/core/ggml_runner.cpp b/src/core/ggml_runner.cpp
index abfa8639bd..106a435870 100644
--- a/src/core/ggml_runner.cpp
+++ b/src/core/ggml_runner.cpp
@@ -21,11 +21,12 @@ ggml_tensor* ggml_ext_attention_ext(GGMLRunnerContext* ctx,
                                     ggml_tensor* mask,
                                     bool skip_reshape,
                                     bool flash_attn,
-                                    float kv_scale) {
+                                    float kv_scale,
+                                    bool* used_flash_attn) {
     if (ctx->attn_scale > 0.f) {
         kv_scale = ctx->attn_scale;
     }
-    return ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, mask, skip_reshape, flash_attn, kv_scale, ctx->sage_attn_enabled);
+    return ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, n_head, mask, skip_reshape, flash_attn, kv_scale, ctx->sage_attn_enabled, used_flash_attn);
 }
 
 void GGMLRunner::alloc_params_ctx() {
@@ -481,6 +482,7 @@ void GGMLRunner::runner_end() {
         return;
     }
     workspace_.release();
+    workspace_.release_blas_scratch();
     cache_.clear();
     logged_compute_bytes_.clear();
     logged_segment_count_ = 0;
@@ -515,9 +517,10 @@ GGMLRunner::~GGMLRunner() {
     free_params_ctx();
 }
 
-GGMLRunnerContext GGMLRunner::get_context() {
+GGMLRunnerContext GGMLRunner::get_context(ggml_cgraph* graph) {
     GGMLRunnerContext runner_ctx;
     runner_ctx.ggml_ctx              = compute_ctx;
+    runner_ctx.graph                 = graph;
     runner_ctx.backend               = runtime_backend;
     runner_ctx.flash_attn_enabled    = flash_attn_enabled;
     runner_ctx.sage_attn_enabled     = sage_attn_enabled;
@@ -532,8 +535,8 @@ GGMLRunnerContext GGMLRunner::get_context() {
     runner_ctx.get_cache_tensor      = [this](const std::string& name) {
         return this->get_cache_tensor_by_name(name);
     };
-    runner_ctx.cache_tensor = [this](const std::string& name, ggml_tensor* tensor) {
-        this->cache(name, tensor);
+    runner_ctx.cache_tensor = [this, graph](const std::string& name, ggml_tensor* tensor) {
+        this->cache(name, tensor, graph);
     };
     runner_ctx.set_backend_tensor_data = [this](ggml_tensor* tensor, const void* data) {
         this->set_backend_tensor_data(tensor, data);
@@ -575,7 +578,7 @@ ggml_tensor* GGMLRunner::to_backend(ggml_tensor* tensor) {
     }
 }
 
-void GGMLRunner::cache(const std::string name, ggml_tensor* tensor) {
+void GGMLRunner::cache(const std::string name, ggml_tensor* tensor, ggml_cgraph* graph) {
     if (tensor != nullptr && tensor->view_src != nullptr) {
         tensor = ggml_cont(compute_ctx, tensor);
     }
@@ -583,6 +586,10 @@ void GGMLRunner::cache(const std::string name, ggml_tensor* tensor) {
         ggml_set_output(tensor);
     }
     cache_.stage(name, tensor);
+    if (graph != nullptr && tensor != nullptr) {
+        // Schedule the cache output here so its source can be reused before graph end.
+        ggml_build_forward_expand(graph, tensor);
+    }
 }
 
 std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
@@ -590,6 +597,7 @@ std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
                                                      bool auto_runner_end,
                                                      bool no_return,
                                                      const std::function<bool()>& read_outputs) {
+    last_compute_status_ = GGML_STATUS_FAILED;
     if (graph_active_) {
         LOG_ERROR("%s does not support reentrant graph execution", get_desc().c_str());
         return std::nullopt;
@@ -613,7 +621,9 @@ std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
         GGMLRunner& runner;
         const bool& success;
         ~GraphEndGuard() {
-            runner.workspace_.segment_end();
+            if (!runner.workspace_.segment_end()) {
+                runner.last_compute_status_ = GGML_STATUS_FAILED;
+            }
             runner.cache_.graph_end(false);
             runner.cut_cache_.clear();
             runner.free_compute_ctx();
@@ -641,7 +651,12 @@ std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
     std::optional<sd::Tensor<float>> output;
     try {
         output = execute_graph(graph, n_threads, no_return, read_outputs);
+    } catch (const std::bad_alloc&) {
+        last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
+        LOG_ERROR("%s graph allocation failed", get_desc().c_str());
+        return std::nullopt;
     } catch (const std::exception& error) {
+        last_compute_status_ = GGML_STATUS_FAILED;
         LOG_ERROR("%s graph execution failed on %s: %s", get_desc().c_str(),
                   ggml_backend_name(runtime_backend), error.what());
         return std::nullopt;
@@ -649,6 +664,7 @@ std::optional<sd::Tensor<float>> GGMLRunner::compute(get_graph_cb_t get_graph,
     success = output.has_value();
     if (success) {
         cache_.graph_end(true);
+        last_compute_status_ = GGML_STATUS_SUCCESS;
     }
     return output;
 }
@@ -725,7 +741,7 @@ std::vector<DeviceMemoryRequest> GGMLRunner::memory_requests(
             }
         }
         requests.push_back({size.backend, reinterpret_cast<uintptr_t>(this), pending,
-                            retained, limit});
+                            retained, limit, reuse_device_query_});
     }
     return requests;
 }
@@ -751,13 +767,17 @@ bool GGMLRunner::execute_segment(ggml_cgraph* graph, int n_threads) {
     if (workspace_.cpu_backend() != nullptr) {
         sd_backend_cpu_set_n_threads(workspace_.cpu_backend(), n_threads);
     }
+    workspace_.set_accelerator_threads(n_threads);
     auto scheduler = workspace_.scheduler();
     ggml_status status;
     if (scheduler != nullptr) {
-        if (sd_get_backend_eval_callback() != nullptr && !multi_device_eval_callback_warned) {
+        // CPU + BLAS keeps every tensor in host memory, so callbacks see the same data as on the plain CPU path.
+        auto eval_callback = workspace_.uses_blas() ? sd_get_backend_eval_callback() : nullptr;
+        if (eval_callback == nullptr && sd_get_backend_eval_callback() != nullptr && !multi_device_eval_callback_warned) {
             LOG_WARN("%s: eval callback is not supported with the backend scheduler; ignoring", get_desc().c_str());
             multi_device_eval_callback_warned = true;
         }
+        ggml_backend_sched_set_eval_callback(scheduler, eval_callback, sd_get_backend_eval_callback_data());
         status = ggml_backend_sched_graph_compute(scheduler, graph);
     } else {
         status = sd_backend_graph_compute_with_eval_callback(runtime_backend, graph,
@@ -766,6 +786,7 @@ bool GGMLRunner::execute_segment(ggml_cgraph* graph, int n_threads) {
     }
     workspace_.synchronize();
     if (status != GGML_STATUS_SUCCESS) {
+        last_compute_status_ = status;
         LOG_ERROR("%s compute failed: %s", get_desc().c_str(), ggml_status_to_string(status));
         return false;
     }
@@ -818,13 +839,23 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
     const auto& cached_plan     = resolve_graph_cut_plan(graph);
     const auto full_measurement = measure(graph, cached_plan.compute_buffer_size);
     if (full_measurement.buffers.empty()) {
+        last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
         return std::nullopt;
     }
+    auto fits_monolithic = [&]() {
+        // Planning headroom absorbs allocation estimate drift; execution keeps the normal limits.
+        constexpr size_t planning_headroom = 128ULL * 1024ULL * 1024ULL;
+        auto requests                      = memory_requests(full_measurement.buffers, cache_.pending_bytes(graph));
+        for (auto& request : requests) {
+            request.pending_allocation_bytes = add_bytes(request.pending_allocation_bytes, planning_headroom);
+        }
+        return fits(requests, params);
+    };
     auto manager         = residency_manager.lock();
     const bool segmented = !is_multi_device() && !sd_backend_is_cpu(runtime_backend) &&
                            manager != nullptr && manager->segmented_compute_enabled() &&
                            cached_plan.valid && cached_plan.has_cuts && cached_plan.segments.size() > 1 &&
-                           !fits(memory_requests(full_measurement.buffers, cache_.pending_bytes(graph)), params);
+                           !fits_monolithic();
     ggml_graph_cut::Plan monolithic_plan;
     if (!segmented) {
         monolithic_plan.segments.emplace_back();
@@ -888,7 +919,9 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
             SegmentGraphBindings& bindings;
             ggml_context* context;
             ~SegmentCleanup() {
-                runner.workspace_.segment_end();
+                if (!runner.workspace_.segment_end()) {
+                    runner.last_compute_status_ = GGML_STATUS_FAILED;
+                }
                 bindings.restore();
                 weights.segment_end();
                 ggml_free(context);
@@ -898,6 +931,7 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
 
         auto measurement = segmented ? measure(segment_graph, segment.compute_buffer_size) : full_measurement;
         if (!workspace_.prepare(measurement)) {
+            last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
             return fail_segment("workspace preparation");
         }
         const size_t cut_bytes       = last ? 0 : cut_cache_.estimate_output_bytes(graph, segment);
@@ -912,7 +946,11 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
                 sync_runtime_residency();
                 requests = memory_requests(measurement.buffers, new_cache_bytes);
             }
-            return weights.ensure_segment_capacity(index, requests);
+            const bool ready = weights.ensure_segment_capacity(index, requests);
+            if (!ready && manager != nullptr) {
+                last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
+            }
+            return ready;
         };
         if (!weights.segment_start(index, ensure_capacity)) {
             return fail_segment("weight preparation");
@@ -921,12 +959,17 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
         if (!workspace_.measurement_matches(segment_graph, measurement)) {
             measurement = measure(segment_graph, segment.compute_buffer_size);
         }
-        if (!workspace_.prepare(measurement) || !ensure_capacity()) {
+        if (!workspace_.prepare(measurement)) {
+            last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
+            return fail_segment("workspace preparation");
+        }
+        if (!ensure_capacity()) {
             return fail_segment("workspace capacity check");
         }
         if (!workspace_.allocate(segment_graph, [&](ggml_backend_sched_t scheduler, ggml_cgraph* current) {
                 pin_multi_device_nodes(scheduler, current);
             })) {
+            last_compute_status_ = GGML_STATUS_ALLOC_FAILED;
             return fail_segment("workspace allocation");
         }
         for (const auto& size : measurement.buffers) {
@@ -945,10 +988,16 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
         }
         LOG_DEBUG("%s executing segment %zu/%zu: %s", get_desc().c_str(),
                   index + 1, plan.segments.size(), segment.group_name.c_str());
-        if (!execute_segment(segment_graph, n_threads) ||
-            !cache_.capture(segment_graph) ||
-            !cut_cache_.capture(graph, segment, get_desc().c_str())) {
-            return fail_segment("execution or output caching");
+        if (!execute_segment(segment_graph, n_threads)) {
+            return fail_segment("execution");
+        }
+        auto cache_status = cache_.capture(segment_graph);
+        if (cache_status == GGML_STATUS_SUCCESS) {
+            cache_status = cut_cache_.capture(graph, segment, get_desc().c_str());
+        }
+        if (cache_status != GGML_STATUS_SUCCESS) {
+            last_compute_status_ = cache_status;
+            return fail_segment("output caching");
         }
         sync_runtime_residency();
         if (last) {
@@ -964,6 +1013,7 @@ std::optional<Tensor<float>> GGMLRunner::execute_graph(ggml_cgraph* graph, int n
             }
         }
         if (!workspace_.segment_end()) {
+            last_compute_status_ = GGML_STATUS_FAILED;
             return fail_segment("workspace synchronization");
         }
         // Final outputs and their callbacks may still be views of consumed cuts.
diff --git a/src/core/ggml_runner.h b/src/core/ggml_runner.h
index 29a6808bd8..5b963982a3 100644
--- a/src/core/ggml_runner.h
+++ b/src/core/ggml_runner.h
@@ -67,6 +67,7 @@ struct WeightAdapter {
 struct GGMLRunnerContext {
     ggml_backend_t backend                                           = nullptr;
     ggml_context* ggml_ctx                                           = nullptr;
+    ggml_cgraph* graph                                               = nullptr;
     bool flash_attn_enabled                                          = false;
     bool sage_attn_enabled                                           = false;
     float linear_scale                                               = 0.f;
@@ -102,6 +103,12 @@ struct GGMLRunnerContext {
         return get_cache_tensor(name);
     }
 
+    void expand_graph(ggml_tensor* tensor) const {
+        if (graph != nullptr && tensor != nullptr) {
+            ggml_build_forward_expand(graph, tensor);
+        }
+    }
+
     void persist_cache_tensor(const std::string& name, ggml_tensor* tensor) const {
         if (!cache_tensor || tensor == nullptr) {
             return;
@@ -122,15 +129,17 @@ ggml_tensor* ggml_ext_attention_ext(GGMLRunnerContext* ctx,
                                     ggml_tensor* k,
                                     ggml_tensor* v,
                                     int64_t n_head,
-                                    ggml_tensor* mask = nullptr,
-                                    bool skip_reshape = false,
-                                    bool flash_attn   = false,
-                                    float kv_scale    = 1.f);
+                                    ggml_tensor* mask     = nullptr,
+                                    bool skip_reshape     = false,
+                                    bool flash_attn       = false,
+                                    float kv_scale        = 1.f,
+                                    bool* used_flash_attn = nullptr);
 
 struct GGMLRunner {
 private:
     std::map<ggml_backend_t, size_t> logged_compute_bytes_;
-    size_t logged_segment_count_ = 0;
+    size_t logged_segment_count_     = 0;
+    ggml_status last_compute_status_ = GGML_STATUS_SUCCESS;
 
     sd::ComputeWorkspace::Measurement measure(ggml_cgraph* graph, size_t direct_bytes);
     std::vector<DeviceMemoryRequest> memory_requests(const std::vector<sd::BackendBufferSize>& sizes,
@@ -156,6 +165,7 @@ struct GGMLRunner {
     bool graph_active_        = false;
 
     size_t max_graph_vram_bytes        = 0;
+    bool reuse_device_query_           = false;
     bool graph_cut_layer_split_enabled = false;
     std::vector<size_t> graph_cut_layer_split_backend_vram_limits_;
 
@@ -288,7 +298,8 @@ struct GGMLRunner {
 
     virtual ~GGMLRunner();
 
-    virtual GGMLRunnerContext get_context();
+    // Binding a graph schedules cache outputs at registration instead of graph end.
+    virtual GGMLRunnerContext get_context(ggml_cgraph* graph = nullptr);
 
     void reset_compute_ctx();
 
@@ -323,7 +334,7 @@ struct GGMLRunner {
 
     ggml_tensor* to_backend(ggml_tensor* tensor);
 
-    void cache(const std::string name, ggml_tensor* tensor);
+    void cache(const std::string name, ggml_tensor* tensor, ggml_cgraph* graph = nullptr);
 
     ggml_tensor* get_cache_tensor_by_name(const std::string& name) {
         return cache_.get(name);
@@ -335,6 +346,8 @@ struct GGMLRunner {
                                              bool no_return                            = false,
                                              const std::function<bool()>& read_outputs = {});
 
+    ggml_status last_compute_status() const { return last_compute_status_; }
+
     void set_flash_attention_enabled(bool enabled) {
         flash_attn_enabled = enabled;
     }
@@ -369,6 +382,11 @@ struct GGMLRunner {
         weight_adapter = adapter;
     }
 
+    // See DeviceMemoryRequest::reuse_device_query; runner_end() drops the saved reading.
+    void set_reuse_device_query(bool enabled) {
+        reuse_device_query_ = enabled;
+    }
+
     void set_max_graph_vram_bytes(size_t max_vram_bytes) {
         max_graph_vram_bytes = max_vram_bytes;
     }
diff --git a/src/core/ggml_tensor_utils.cpp b/src/core/ggml_tensor_utils.cpp
index 1747678d4c..5639c7ed98 100644
--- a/src/core/ggml_tensor_utils.cpp
+++ b/src/core/ggml_tensor_utils.cpp
@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <cstdlib>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 
 #include "core/ggml_extend_backend.h"
@@ -100,7 +101,7 @@ void ggml_ext_tensor_diff(
 }
 
 ggml_tensor* load_tensor_from_file(ggml_context* ctx, const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         LOG_ERROR("failed to open '%s'", file_path.c_str());
         return nullptr;
diff --git a/src/core/layer_split_partition.cpp b/src/core/layer_split_partition.cpp
index e19f47b261..b528f1baf2 100644
--- a/src/core/layer_split_partition.cpp
+++ b/src/core/layer_split_partition.cpp
@@ -7,6 +7,7 @@
 #include <unordered_set>
 #include <utility>
 
+#include "core/layer_split_ranges.h"
 #include "core/util.h"
 
 namespace sd {
@@ -84,14 +85,20 @@ namespace sd {
             if (dev != nullptr) {
                 ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
             }
-            if (free_bytes > 0) {
-                capacities[i] = std::max<int64_t>((int64_t)free_bytes - compute_headroom_bytes, 0);
-            }
             size_t limit_bytes = graph_cut_layer_split_backend_vram_limit(backend_vram_limits,
                                                                           i,
                                                                           primary_backend_vram_limit);
+            // The headroom comes off whichever bound applies: an explicit --max-vram budget also has to hold the
+            // compute buffers of the segments this device runs, so weights alone may not fill it.
+            int64_t bound = -1;
+            if (free_bytes > 0) {
+                bound = (int64_t)free_bytes;
+            }
             if (limit_bytes > 0) {
-                capacities[i] = std::min<int64_t>(capacities[i], (int64_t)limit_bytes);
+                bound = bound < 0 ? (int64_t)limit_bytes : std::min<int64_t>(bound, (int64_t)limit_bytes);
+            }
+            if (bound >= 0) {
+                capacities[i] = std::max<int64_t>(bound - compute_headroom_bytes, 0);
             }
         }
         return capacities;
@@ -151,27 +158,33 @@ namespace sd {
             return param_assignments.count(param) != 0;
         });
 
-        std::vector<ggml_backend_t> backend_by_segment(plan.segments.size(), split_backends[0]);
-        size_t current_backend = 0;
-        int64_t current_used   = 0;
-        for (size_t seg_idx = 0; seg_idx < plan.segments.size(); seg_idx++) {
-            int64_t bytes = segment_param_bytes[seg_idx];
-            while (!reuse_assignments && current_backend + 1 < split_backends.size() &&
-                   bytes > 0 &&
-                   current_used + bytes > backend_capacities[current_backend]) {
-                current_backend++;
-                current_used = 0;
-            }
-            if (!reuse_assignments && bytes > 0 && current_used + bytes > backend_capacities[current_backend]) {
+        LayerSplitRangesResult ranges;
+        if (!reuse_assignments) {
+            ranges = plan_layer_split_ranges(segment_param_bytes, backend_capacities);
+            if (!ranges.ok) {
                 LOG_ERROR("%s graph-cut layer split: segment %zu needs %.1f MB on %s, but only %.1f MB is available under current VRAM limits",
                           desc,
-                          seg_idx,
-                          (current_used + bytes) / (1024.0 * 1024.0),
-                          layer_split_backend_device_display_name(split_backends[current_backend]).c_str(),
-                          backend_capacities[current_backend] / (1024.0 * 1024.0));
+                          ranges.failed_segment,
+                          ranges.failed_need / (1024.0 * 1024.0),
+                          layer_split_backend_device_display_name(split_backends[ranges.failed_device]).c_str(),
+                          backend_capacities[ranges.failed_device] / (1024.0 * 1024.0));
                 return false;
             }
-            current_used += bytes;
+            for (size_t i = 0; i < split_backends.size(); i++) {
+                LOG_INFO("%s graph-cut layer split: %s weight capacity %.1f MB (%s)",
+                         desc,
+                         layer_split_backend_device_display_name(split_backends[i]).c_str(),
+                         backend_capacities[i] / (1024.0 * 1024.0),
+                         ranges.used_fill_first ? "fill-first plan, proportional plan did not fit" : "proportional plan");
+            }
+        }
+
+        std::vector<ggml_backend_t> backend_by_segment(plan.segments.size(), split_backends[0]);
+        size_t current_backend = 0;
+        for (size_t seg_idx = 0; seg_idx < plan.segments.size(); seg_idx++) {
+            if (!reuse_assignments) {
+                current_backend = ranges.device_by_segment[seg_idx];
+            }
 
             for (ggml_tensor* param : segment_params[seg_idx]) {
                 ggml_backend_t target_backend = split_backends[current_backend];
diff --git a/src/core/layer_split_ranges.h b/src/core/layer_split_ranges.h
new file mode 100644
index 0000000000..2aea3aa75b
--- /dev/null
+++ b/src/core/layer_split_ranges.h
@@ -0,0 +1,124 @@
+#ifndef __SD_CORE_LAYER_SPLIT_RANGES_H__
+#define __SD_CORE_LAYER_SPLIT_RANGES_H__
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <vector>
+
+namespace sd {
+
+    struct LayerSplitRangesResult {
+        bool ok = false;
+        // Device index per segment (contiguous, non-decreasing).
+        std::vector<size_t> device_by_segment;
+        // Weight bytes placed on each device.
+        std::vector<int64_t> bytes_by_device;
+        // On failure: the segment that did not fit, the device it was tried on and the bytes that device would need.
+        size_t failed_segment = 0;
+        size_t failed_device  = 0;
+        int64_t failed_need   = 0;
+        // True when the proportional plan did not fit and the fill-first plan was used instead.
+        bool used_fill_first = false;
+    };
+
+    // Fill-first: each device takes segments until the next one would exceed its capacity.
+    inline LayerSplitRangesResult plan_layer_split_ranges_fill_first(const std::vector<int64_t>& segment_bytes,
+                                                                     const std::vector<int64_t>& capacities) {
+        LayerSplitRangesResult r;
+        r.device_by_segment.assign(segment_bytes.size(), 0);
+        r.bytes_by_device.assign(capacities.size(), 0);
+        if (capacities.empty()) {
+            return r;
+        }
+        size_t dev   = 0;
+        int64_t used = 0;
+        for (size_t s = 0; s < segment_bytes.size(); s++) {
+            const int64_t bytes = segment_bytes[s];
+            while (dev + 1 < capacities.size() && bytes > 0 && used + bytes > capacities[dev]) {
+                dev++;
+                used = 0;
+            }
+            if (bytes > 0 && used + bytes > capacities[dev]) {
+                r.failed_segment = s;
+                r.failed_device  = dev;
+                r.failed_need    = used + bytes;
+                return r;
+            }
+            used += bytes;
+            r.device_by_segment[s] = dev;
+            r.bytes_by_device[dev] += bytes;
+        }
+        r.ok = true;
+        return r;
+    }
+
+    // Contiguous ranges sized in proportion to each device's capacity, so every device keeps the same fraction of its
+    // budget free for the compute buffers of the segments it runs. A fill-first plan packs the main device up to its
+    // weight capacity and leaves its compute buffers no room; it is only used when the proportional plan does not fit.
+    inline LayerSplitRangesResult plan_layer_split_ranges(const std::vector<int64_t>& segment_bytes,
+                                                          const std::vector<int64_t>& capacities) {
+        LayerSplitRangesResult r;
+        r.device_by_segment.assign(segment_bytes.size(), 0);
+        r.bytes_by_device.assign(capacities.size(), 0);
+        if (capacities.empty()) {
+            return r;
+        }
+        double total = 0.0;
+        for (int64_t b : segment_bytes) {
+            total += (double)(b > 0 ? b : 0);
+        }
+        double capacity_sum = 0.0;
+        std::vector<double> caps(capacities.size(), 0.0);
+        for (size_t i = 0; i < capacities.size(); i++) {
+            // An unknown capacity (very large sentinel) counts as able to hold everything.
+            caps[i] = capacities[i] > 0 ? std::min((double)capacities[i], total) : 0.0;
+            capacity_sum += caps[i];
+        }
+        if (capacities.size() == 1 || total <= 0.0 || capacity_sum <= 0.0) {
+            return plan_layer_split_ranges_fill_first(segment_bytes, capacities);
+        }
+
+        // Cumulative byte boundary at which each device's range ends.
+        std::vector<double> boundary(capacities.size(), total);
+        double acc = 0.0;
+        for (size_t i = 0; i + 1 < capacities.size(); i++) {
+            acc += total * caps[i] / capacity_sum;
+            boundary[i] = acc;
+        }
+
+        size_t dev      = 0;
+        int64_t used    = 0;
+        double consumed = 0.0;
+        bool fits       = true;
+        for (size_t s = 0; s < segment_bytes.size(); s++) {
+            const int64_t bytes = segment_bytes[s];
+            if (bytes > 0) {
+                // Move on once this segment's midpoint lies past the current device's share, or it would not fit.
+                while (dev + 1 < capacities.size() &&
+                       (consumed + bytes * 0.5 > boundary[dev] || used + bytes > capacities[dev])) {
+                    dev++;
+                    used = 0;
+                }
+                if (used + bytes > capacities[dev]) {
+                    fits = false;
+                    break;
+                }
+            }
+            used += bytes;
+            consumed += (double)(bytes > 0 ? bytes : 0);
+            r.device_by_segment[s] = dev;
+            r.bytes_by_device[dev] += bytes;
+        }
+        if (fits) {
+            r.ok = true;
+            return r;
+        }
+        LayerSplitRangesResult fallback = plan_layer_split_ranges_fill_first(segment_bytes, capacities);
+        fallback.used_fill_first        = true;
+        return fallback;
+    }
+
+}  // namespace sd
+
+#endif  // __SD_CORE_LAYER_SPLIT_RANGES_H__
diff --git a/src/core/probe/backend_probe.cpp b/src/core/probe/backend_probe.cpp
new file mode 100644
index 0000000000..57a90ad64e
--- /dev/null
+++ b/src/core/probe/backend_probe.cpp
@@ -0,0 +1,193 @@
+#include "core/probe/backend_probe.h"
+
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <exception>
+
+#include "core/ggml_extend_backend.h"
+#include "core/util.h"
+#include "ggml-alloc.h"
+
+BackendProbeGraph::BackendProbeGraph(ggml_backend_t backend, size_t graph_size)
+    : backend_(backend), graph_size_(graph_size) {
+    ctx = ggml_init({graph_size * ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false), nullptr, true});
+}
+
+BackendProbeGraph::~BackendProbeGraph() {
+    ggml_backend_buffer_free(buffer_);
+    ggml_free(ctx);
+}
+
+bool BackendProbeGraph::supported() {
+    if (backend_ == nullptr || ctx == nullptr || output == nullptr) {
+        return false;
+    }
+    if (graph_ == nullptr) {
+        graph_ = ggml_new_graph_custom(ctx, graph_size_, false);
+        ggml_build_forward_expand(graph_, output);
+    }
+    for (int i = 0; i < ggml_graph_n_nodes(graph_); ++i) {
+        if (!ggml_backend_supports_op(backend_, ggml_graph_node(graph_, i))) {
+            return false;
+        }
+    }
+    return true;
+}
+
+bool BackendProbeGraph::allocate() {
+    if (ctx == nullptr || backend_ == nullptr) {
+        return false;
+    }
+    if (buffer_ == nullptr) {
+        buffer_ = ggml_backend_alloc_ctx_tensors(ctx, backend_);
+    }
+    return buffer_ != nullptr;
+}
+
+bool BackendProbeGraph::initialize_f32_inputs() {
+    uint32_t state = 42;
+    for (auto* tensor : inputs) {
+        if (tensor == nullptr || tensor->type != GGML_TYPE_F32 || !ggml_is_contiguous(tensor)) {
+            return false;
+        }
+        std::vector<float> values(static_cast<size_t>(ggml_nelements(tensor)));
+        for (float& value : values) {
+            state = state * 1664525u + 1013904223u;
+            value = static_cast<float>(state >> 8) / 16777216.f - 0.5f;
+        }
+        ggml_backend_tensor_set(tensor, values.data(), 0, ggml_nbytes(tensor));
+    }
+    return true;
+}
+
+double BackendProbeGraph::run() {
+    if (graph_ == nullptr || buffer_ == nullptr) {
+        return -1.;
+    }
+    ggml_backend_synchronize(backend_);
+    const auto start  = std::chrono::steady_clock::now();
+    const auto status = ggml_backend_graph_compute(backend_, graph_);
+    ggml_backend_synchronize(backend_);
+    if (status != GGML_STATUS_SUCCESS) {
+        return -1.;
+    }
+    return std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - start).count();
+}
+
+static bool backend_probe_outputs_match(const BackendProbeGraph& reference,
+                                        const BackendProbeGraph& candidate,
+                                        const BackendProbeOptions& options) {
+    if (!ggml_are_same_shape(reference.output, candidate.output) ||
+        reference.output->type != GGML_TYPE_F32 || candidate.output->type != GGML_TYPE_F32 ||
+        !ggml_is_contiguous(reference.output) || !ggml_is_contiguous(candidate.output)) {
+        return false;
+    }
+    const size_t count = static_cast<size_t>(ggml_nelements(reference.output));
+    std::vector<float> expected(count), actual(count);
+    ggml_backend_tensor_get(reference.output, expected.data(), 0, count * sizeof(float));
+    ggml_backend_tensor_get(candidate.output, actual.data(), 0, count * sizeof(float));
+    for (size_t i = 0; i < count; ++i) {
+        if (!std::isfinite(expected[i]) || !std::isfinite(actual[i]) ||
+            std::abs(expected[i] - actual[i]) > options.absolute_tolerance + options.relative_tolerance * std::abs(expected[i])) {
+            return false;
+        }
+    }
+    return true;
+}
+
+BackendProbeResult sd_run_backend_probe(ggml_backend_t backend,
+                                        int n_threads,
+                                        const std::vector<BackendProbeCase>& cases,
+                                        const BackendProbeOptions& options) {
+    BackendProbeResult result;
+    if (backend == nullptr || cases.empty() || options.graph_size == 0 || options.warmup_runs < 1 || options.measurement_runs < 1 ||
+        !std::isfinite(options.time_budget_ms) || options.time_budget_ms < 0. ||
+        !std::isfinite(options.min_improvement) || options.min_improvement < 0. || options.min_improvement >= 1. ||
+        !std::isfinite(options.absolute_tolerance) || options.absolute_tolerance < 0.f ||
+        !std::isfinite(options.relative_tolerance) || options.relative_tolerance < 0.f) {
+        result.error = "invalid probe configuration";
+        return result;
+    }
+    try {
+        if (sd_backend_is_cpu(backend)) {
+            sd_backend_cpu_set_n_threads(backend, n_threads > 0 ? n_threads : sd_get_num_physical_cores());
+        }
+        auto dev = ggml_backend_get_device(backend);
+        if (dev != nullptr && !sd_backend_is_cpu(backend)) {
+            size_t free_bytes = 0, total_bytes = 0;
+            ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
+            if (free_bytes > total_bytes || free_bytes < options.min_free_device_bytes) {
+                result.error = "insufficient probe memory";
+                return result;
+            }
+        }
+        double timed_ms = 0.;
+        for (const auto& test : cases) {
+            BackendProbeGraph reference(backend, options.graph_size), candidate(backend, options.graph_size);
+            if (!test.build || reference.ctx == nullptr || candidate.ctx == nullptr) {
+                result.error = "probe graph unavailable";
+                return result;
+            }
+            test.build(reference, false);
+            test.build(candidate, true);
+            if (!reference.supported() || !candidate.supported()) {
+                result.error = "unsupported probe graph";
+                return result;
+            }
+            if (!reference.allocate() || !candidate.allocate()) {
+                result.error = "probe allocation failed";
+                return result;
+            }
+            auto initialize = [&](BackendProbeGraph& graph) {
+                return test.initialize ? test.initialize(graph) : graph.initialize_f32_inputs();
+            };
+            if (!initialize(reference) || !initialize(candidate)) {
+                result.error = "probe input initialization failed";
+                return result;
+            }
+            for (int i = 0; i < options.warmup_runs; ++i) {
+                if (reference.run() < 0. || candidate.run() < 0.) {
+                    result.error = "probe warmup failed";
+                    return result;
+                }
+            }
+            if (!(test.compare ? test.compare(reference, candidate) : backend_probe_outputs_match(reference, candidate, options))) {
+                result.error = "probe output check failed";
+                return result;
+            }
+            std::vector<double> reference_times(options.measurement_runs), candidate_times(options.measurement_runs);
+            for (int i = 0; i < options.measurement_runs; ++i) {
+                if (timed_ms >= options.time_budget_ms) {
+                    result.error = "probe time budget exhausted";
+                    return result;
+                }
+                // Alternate order so clock ramp-up does not consistently favor one path.
+                if (i % 2 == 0) {
+                    reference_times[i] = reference.run();
+                    candidate_times[i] = candidate.run();
+                } else {
+                    candidate_times[i] = candidate.run();
+                    reference_times[i] = reference.run();
+                }
+                if (reference_times[i] < 0. || candidate_times[i] < 0.) {
+                    result.error = "probe execution failed";
+                    return result;
+                }
+                timed_ms += reference_times[i] + candidate_times[i];
+            }
+            auto median = [](std::vector<double>& values) {
+                std::sort(values.begin(), values.end());
+                const size_t mid = values.size() / 2;
+                return values.size() % 2 ? values[mid] : (values[mid - 1] + values[mid]) * 0.5;
+            };
+            result.reference_ms += median(reference_times);
+            result.candidate_ms += median(candidate_times);
+        }
+        result.completed        = true;
+        result.candidate_faster = result.candidate_ms < result.reference_ms * (1. - options.min_improvement);
+    } catch (const std::exception& error) {
+        result.error = error.what();
+    }
+    return result;
+}
diff --git a/src/core/probe/backend_probe.h b/src/core/probe/backend_probe.h
new file mode 100644
index 0000000000..55b27762bd
--- /dev/null
+++ b/src/core/probe/backend_probe.h
@@ -0,0 +1,64 @@
+#ifndef __SD_CORE_PROBE_BACKEND_PROBE_H__
+#define __SD_CORE_PROBE_BACKEND_PROBE_H__
+
+#include <functional>
+#include <string>
+#include <vector>
+
+#include "ggml-backend.h"
+
+struct BackendProbeOptions {
+    size_t graph_size            = 128;
+    int warmup_runs              = 1;
+    int measurement_runs         = 5;
+    double time_budget_ms        = 1000.;
+    size_t min_free_device_bytes = 128ULL * 1024 * 1024;
+    double min_improvement       = 0.05;
+    float absolute_tolerance     = 0.001f;
+    float relative_tolerance     = 0.02f;
+};
+
+class BackendProbeGraph {
+    ggml_backend_t backend_;
+    size_t graph_size_;
+    ggml_backend_buffer_t buffer_ = nullptr;
+    ggml_cgraph* graph_           = nullptr;
+
+public:
+    ggml_context* ctx = nullptr;
+    std::vector<ggml_tensor*> inputs;
+    ggml_tensor* output = nullptr;
+
+    explicit BackendProbeGraph(ggml_backend_t backend, size_t graph_size = 128);
+    ~BackendProbeGraph();
+    BackendProbeGraph(const BackendProbeGraph&)            = delete;
+    BackendProbeGraph& operator=(const BackendProbeGraph&) = delete;
+
+    bool supported();
+    bool allocate();
+    bool initialize_f32_inputs();
+    double run();
+};
+
+struct BackendProbeCase {
+    // Variants must be repeatable without mutating inputs; leave output null if unavailable.
+    std::function<void(BackendProbeGraph&, bool candidate)> build;
+    // Optional overrides for non-F32 inputs/outputs or operation-specific data.
+    std::function<bool(BackendProbeGraph&)> initialize;
+    std::function<bool(const BackendProbeGraph&, const BackendProbeGraph&)> compare;
+};
+
+struct BackendProbeResult {
+    bool completed        = false;
+    bool candidate_faster = false;
+    double reference_ms   = 0.;
+    double candidate_ms   = 0.;
+    std::string error;
+};
+
+BackendProbeResult sd_run_backend_probe(ggml_backend_t backend,
+                                        int n_threads,
+                                        const std::vector<BackendProbeCase>& cases,
+                                        const BackendProbeOptions& options = {});
+
+#endif  // __SD_CORE_PROBE_BACKEND_PROBE_H__
diff --git a/src/core/probe/conv_direct.cpp b/src/core/probe/conv_direct.cpp
new file mode 100644
index 0000000000..7f4666a41f
--- /dev/null
+++ b/src/core/probe/conv_direct.cpp
@@ -0,0 +1,96 @@
+#include "core/probe/conv_direct.h"
+
+#include <cmath>
+#include <exception>
+#include <utility>
+#include <vector>
+
+#include "core/ggml_extend_backend.h"
+#include "core/probe/backend_probe.h"
+#include "core/util.h"
+
+static void build_conv_direct_probe(BackendProbeGraph& graph, bool conv3d, bool direct, int size = 32, int channels = 64) {
+    if (conv3d) {
+        auto input   = ggml_new_tensor_4d(graph.ctx, GGML_TYPE_F32, size / 2, size / 2, 4, channels);
+        auto weight  = ggml_new_tensor_4d(graph.ctx, GGML_TYPE_F16, 3, 3, 3, channels * channels);
+        graph.inputs = {input, weight};
+        graph.output = direct ? ggml_conv_3d_direct(graph.ctx, weight, input, 1, 1, 1, 1, 1, 1, 1, 1, 1, channels, 1, channels)
+                              : ggml_conv_3d(graph.ctx, weight, input, channels, 1, 1, 1, 1, 1, 1, 1, 1, 1);
+    } else {
+        auto input   = ggml_new_tensor_4d(graph.ctx, GGML_TYPE_F32, size, size, channels, 1);
+        auto weight  = ggml_new_tensor_4d(graph.ctx, GGML_TYPE_F16, 3, 3, channels, channels);
+        graph.inputs = {input, weight};
+        graph.output = direct ? ggml_conv_2d_direct(graph.ctx, weight, input, 1, 1, 1, 1, 1, 1)
+                              : ggml_conv_2d(graph.ctx, weight, input, 1, 1, 1, 1, 1, 1);
+    }
+}
+
+static bool initialize_conv_probe(BackendProbeGraph& graph, bool conv3d) {
+    uint32_t state = 42;
+    for (auto* tensor : graph.inputs) {
+        const size_t count = static_cast<size_t>(ggml_nelements(tensor));
+        const float scale  = tensor->type == GGML_TYPE_F16 ? 1.f / std::sqrt(float(graph.inputs[0]->ne[conv3d ? 3 : 2] * (conv3d ? 27 : 9))) : 1.f;
+        std::vector<ggml_fp16_t> values(count);
+        for (auto& value : values) {
+            state = state * 1664525u + 1013904223u;
+            value = ggml_fp32_to_fp16((static_cast<float>(state >> 8) / 16777216.f - 0.5f) * scale);
+        }
+        if (tensor->type == GGML_TYPE_F16) {
+            ggml_backend_tensor_set(tensor, values.data(), 0, count * sizeof(ggml_fp16_t));
+        } else {
+            // im2col rounds inputs to F16; use the same input precision for both paths.
+            std::vector<float> input(count);
+            ggml_fp16_to_fp32_row(values.data(), input.data(), count);
+            ggml_backend_tensor_set(tensor, input.data(), 0, count * sizeof(float));
+        }
+    }
+    return true;
+}
+
+bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d) {
+    if (backend == nullptr) {
+        return false;
+    }
+    const char* name = ggml_backend_name(backend);
+    const int dims   = conv3d ? 3 : 2;
+    try {
+        BackendProbeGraph capability(backend);
+        if (capability.ctx == nullptr) {
+            LOG_INFO("conv%dd direct auto: %s -> off (probe graph unavailable)", dims, name);
+            return false;
+        }
+        build_conv_direct_probe(capability, conv3d, true);
+        if (!capability.supported()) {
+            LOG_INFO("conv%dd direct auto: %s -> off (unsupported probe graph)", dims, name);
+            return false;
+        }
+        if (sd_backend_is(backend, "Vulkan")) {
+            std::vector<BackendProbeCase> cases;
+            for (const auto& shape : {std::pair<int, int>{32, 64}, {16, 128}}) {
+                BackendProbeCase test;
+                test.build = [conv3d, shape](BackendProbeGraph& graph, bool direct) {
+                    build_conv_direct_probe(graph, conv3d, direct, shape.first, shape.second);
+                };
+                test.initialize = [conv3d](BackendProbeGraph& graph) { return initialize_conv_probe(graph, conv3d); };
+                cases.push_back(std::move(test));
+            }
+            const auto result = sd_run_backend_probe(backend, 0, cases);
+            if (!result.completed) {
+                LOG_WARN("conv%dd direct auto: %s -> off (%s)", dims, name, result.error.c_str());
+                return false;
+            }
+            LOG_INFO("conv%dd direct auto: %s -> %s (probe: im2col %.3f ms, direct %.3f ms)",
+                     dims, name, result.candidate_faster ? "on" : "off", result.reference_ms, result.candidate_ms);
+            return result.candidate_faster;
+        }
+        if (sd_backend_supports_conv_implicit_gemm(backend, conv3d)) {
+            LOG_INFO("conv%dd direct auto: %s -> on (implicit GEMM)", dims, name);
+            return true;
+        }
+        LOG_INFO("conv%dd direct auto: %s -> off (no implicit GEMM path)", dims, name);
+        return false;
+    } catch (const std::exception& error) {
+        LOG_WARN("conv%dd direct auto: %s -> off (probe failed: %s)", dims, name, error.what());
+        return false;
+    }
+}
diff --git a/src/core/probe/conv_direct.h b/src/core/probe/conv_direct.h
new file mode 100644
index 0000000000..7b44f83c82
--- /dev/null
+++ b/src/core/probe/conv_direct.h
@@ -0,0 +1,8 @@
+#ifndef __SD_CORE_PROBE_CONV_DIRECT_H__
+#define __SD_CORE_PROBE_CONV_DIRECT_H__
+
+#include "ggml-backend.h"
+
+bool sd_probe_conv_direct(ggml_backend_t backend, bool conv3d);
+
+#endif  // __SD_CORE_PROBE_CONV_DIRECT_H__
diff --git a/src/core/probe/flash_attn.cpp b/src/core/probe/flash_attn.cpp
new file mode 100644
index 0000000000..3dcb95e826
--- /dev/null
+++ b/src/core/probe/flash_attn.cpp
@@ -0,0 +1,101 @@
+#include "core/probe/flash_attn.h"
+
+#include <array>
+#include <cstring>
+#include <exception>
+#include <sstream>
+#include <utility>
+#include <vector>
+
+#include "core/ggml_extend.h"
+#include "core/ggml_extend_backend.h"
+#include "core/probe/backend_probe.h"
+#include "core/util.h"
+
+static bool is_vulkan_nvidia_rtx(ggml_backend_t backend) {
+    auto device = ggml_backend_get_device(backend);
+    if (device == nullptr) {
+        return false;
+    }
+    auto reg = ggml_backend_dev_backend_reg(device);
+    if (reg == nullptr || std::strcmp(ggml_backend_reg_name(reg), "Vulkan") != 0) {
+        return false;
+    }
+    const char* description = ggml_backend_dev_description(device);
+    if (description == nullptr) {
+        return false;
+    }
+    std::istringstream stream(description);
+    std::string token;
+    bool nvidia = false, rtx = false;
+    while (stream >> token) {
+        nvidia = nvidia || token == "NVIDIA";
+        rtx    = rtx || token == "RTX";
+    }
+    return nvidia && rtx;
+}
+
+static void build_flash_attention_probe(BackendProbeGraph& graph, ggml_backend_t backend, int length, int head_size, bool flash) {
+    constexpr int heads = 4;
+    auto q              = ggml_new_tensor_3d(graph.ctx, GGML_TYPE_F32, head_size * heads, length, 1);
+    auto k              = ggml_dup_tensor(graph.ctx, q);
+    auto v              = ggml_dup_tensor(graph.ctx, q);
+    graph.inputs        = {q, k, v};
+    bool used_flash     = false;
+    auto output         = ggml_ext_attention_ext(graph.ctx, backend, q, k, v, heads, nullptr, false, flash, 1.f, false, &used_flash);
+    graph.output        = flash && !used_flash ? nullptr : output;
+}
+
+bool sd_probe_flash_attention(ggml_backend_t backend, int n_threads) {
+    if (backend == nullptr) {
+        return false;
+    }
+    const char* name = ggml_backend_name(backend);
+    try {
+        BackendProbeGraph capability(backend);
+        if (capability.ctx == nullptr) {
+            LOG_INFO("flash attention auto: %s -> off (probe graph unavailable)", name);
+            return false;
+        }
+        build_flash_attention_probe(capability, backend, 512, 64, true);
+        if (!capability.supported()) {
+            LOG_INFO("flash attention auto: %s -> off (unsupported probe graph)", name);
+            return false;
+        }
+        if (sd_backend_is_cpu(backend)) {
+            LOG_INFO("flash attention auto: %s -> on (CPU)", name);
+            return true;
+        }
+        if (sd_backend_supports_cuda_mma(backend)) {
+            LOG_INFO("flash attention auto: %s -> on (CUDA MMA)", name);
+            return true;
+        }
+        if (sd_backend_supports_hip_mma(backend)) {
+            LOG_INFO("flash attention auto: %s -> on (HIP MMA)", name);
+            return true;
+        }
+        if (is_vulkan_nvidia_rtx(backend)) {
+            LOG_INFO("flash attention auto: %s -> on (Vulkan NVIDIA RTX)", name);
+            return true;
+        }
+        std::vector<BackendProbeCase> cases;
+        for (const auto& shape : {std::array<int, 2>{512, 64}, std::array<int, 2>{1024, 128}}) {
+            BackendProbeCase test;
+            test.build = [backend, shape](BackendProbeGraph& graph, bool candidate) {
+                build_flash_attention_probe(graph, backend, shape[0], shape[1], candidate);
+            };
+            cases.push_back(std::move(test));
+        }
+        const auto result = sd_run_backend_probe(backend, n_threads, cases);
+        if (!result.completed) {
+            LOG_WARN("flash attention auto: %s -> off (%s)", name, result.error.c_str());
+            return false;
+        }
+        LOG_INFO("flash attention auto: %s -> %s (probe: normal %.3f ms, flash %.3f ms)",
+                 name, result.candidate_faster ? "on" : "off", result.reference_ms, result.candidate_ms);
+        return result.candidate_faster;
+    } catch (const std::exception& error) {
+        LOG_WARN("flash attention auto: %s -> off (probe failed: %s)", name, error.what());
+        return false;
+    }
+}
diff --git a/src/core/probe/flash_attn.h b/src/core/probe/flash_attn.h
new file mode 100644
index 0000000000..c13083a2a9
--- /dev/null
+++ b/src/core/probe/flash_attn.h
@@ -0,0 +1,8 @@
+#ifndef __SD_CORE_PROBE_FLASH_ATTN_H__
+#define __SD_CORE_PROBE_FLASH_ATTN_H__
+
+#include "ggml-backend.h"
+
+bool sd_probe_flash_attention(ggml_backend_t backend, int n_threads);
+
+#endif  // __SD_CORE_PROBE_FLASH_ATTN_H__
diff --git a/src/core/runner_cache.cpp b/src/core/runner_cache.cpp
index 095621068b..b640bb45a6 100644
--- a/src/core/runner_cache.cpp
+++ b/src/core/runner_cache.cpp
@@ -26,10 +26,13 @@ namespace sd {
 
     std::unique_ptr<CachedTensor> CachedTensor::copy(ggml_backend_t backend,
                                                      const std::string& name,
-                                                     ggml_tensor* source) {
+                                                     ggml_tensor* source,
+                                                     ggml_status& status) {
+        status = GGML_STATUS_FAILED;
         if (ggml_graph_cut::tensor_buffer(source) == nullptr) {
             return nullptr;
         }
+        status         = GGML_STATUS_ALLOC_FAILED;
         auto entry     = std::make_unique<CachedTensor>();
         entry->context = ggml_init({2 * ggml_tensor_overhead(), nullptr, true});
         if (entry->context == nullptr) {
@@ -50,6 +53,7 @@ namespace sd {
         } else {
             ggml_backend_tensor_copy(source, entry->tensor);
         }
+        status = GGML_STATUS_SUCCESS;
         return entry;
     }
 
@@ -106,9 +110,9 @@ namespace sd {
         return pending > SIZE_MAX - committed ? SIZE_MAX : committed + pending;
     }
 
-    bool RunnerCache::capture(ggml_cgraph* graph) {
+    ggml_status RunnerCache::capture(ggml_cgraph* graph) {
         if (outputs_.empty()) {
-            return true;
+            return GGML_STATUS_SUCCESS;
         }
         const auto tensors = cache_graph_tensors(graph);
         for (const auto& output : outputs_) {
@@ -116,14 +120,15 @@ namespace sd {
                 continue;
             }
             GGML_ASSERT(ggml_is_contiguous(output.second));
-            auto entry = CachedTensor::copy(backend_, output.first, output.second);
+            ggml_status status;
+            auto entry = CachedTensor::copy(backend_, output.first, output.second, status);
             if (entry == nullptr) {
-                return false;
+                return status;
             }
             pending_[output.first] = std::move(entry);
         }
         ggml_backend_synchronize(backend_);
-        return true;
+        return GGML_STATUS_SUCCESS;
     }
 
     void RunnerCache::graph_end(bool success) {
@@ -180,9 +185,9 @@ namespace sd {
         }
     }
 
-    bool GraphCutTensorCache::capture(ggml_cgraph* graph,
-                                      const ggml_graph_cut::Segment& segment,
-                                      const char* log_desc) {
+    ggml_status GraphCutTensorCache::capture(ggml_cgraph* graph,
+                                             const ggml_graph_cut::Segment& segment,
+                                             const char* log_desc) {
         size_t copied_bytes = 0;
         size_t copied_count = 0;
         for (int index : segment.output_node_indices) {
@@ -191,10 +196,11 @@ namespace sd {
                 !segment.future_cut_names.count(output->name)) {
                 continue;
             }
-            auto entry = CachedTensor::copy(backend_, output->name, ggml_graph_cut::cache_source_tensor(output));
+            ggml_status status;
+            auto entry = CachedTensor::copy(backend_, output->name, ggml_graph_cut::cache_source_tensor(output), status);
             if (entry == nullptr) {
                 LOG_ERROR("%s failed to capture graph cut tensor: %s", log_desc, output->name);
-                return false;
+                return status;
             }
             const size_t size = ggml_backend_buffer_get_size(entry->buffer);
             copied_bytes      = size > SIZE_MAX - copied_bytes ? SIZE_MAX : copied_bytes + size;
@@ -206,6 +212,6 @@ namespace sd {
             LOG_DEBUG("%s graph cut cache added %6.2f MB (%zu tensors)",
                       log_desc, copied_bytes / (1024.f * 1024.f), copied_count);
         }
-        return true;
+        return GGML_STATUS_SUCCESS;
     }
 }
diff --git a/src/core/runner_cache.h b/src/core/runner_cache.h
index bd331a4bbf..f1b5cf91c1 100644
--- a/src/core/runner_cache.h
+++ b/src/core/runner_cache.h
@@ -20,7 +20,8 @@ namespace sd {
         ~CachedTensor();
         static std::unique_ptr<CachedTensor> copy(ggml_backend_t backend,
                                                   const std::string& name,
-                                                  ggml_tensor* source);
+                                                  ggml_tensor* source,
+                                                  ggml_status& status);
     };
     using CachedTensors = std::map<std::string, std::unique_ptr<CachedTensor>>;
 
@@ -41,7 +42,8 @@ namespace sd {
         const std::map<std::string, ggml_tensor*>& outputs() const { return outputs_; }
         size_t pending_bytes(ggml_cgraph* graph) const;
         size_t resident_bytes(ggml_backend_dev_t device) const;
-        bool capture(ggml_cgraph* graph);
+        bool empty() const { return committed_.empty(); }
+        ggml_status capture(ggml_cgraph* graph);
         void graph_end(bool success);
         void clear();
     };
@@ -57,7 +59,7 @@ namespace sd {
         size_t resident_bytes(ggml_backend_dev_t device) const;
         size_t estimate_output_bytes(ggml_cgraph* graph,
                                      const ggml_graph_cut::Segment& segment) const;
-        bool capture(ggml_cgraph* graph, const ggml_graph_cut::Segment& segment, const char* log_desc);
+        ggml_status capture(ggml_cgraph* graph, const ggml_graph_cut::Segment& segment, const char* log_desc);
         void prune(const std::unordered_set<std::string>& keep_names);
         void clear() { tensors_.clear(); }
     };
diff --git a/src/core/tensor_ggml.hpp b/src/core/tensor_ggml.hpp
index 66ea22206d..5082ce2071 100644
--- a/src/core/tensor_ggml.hpp
+++ b/src/core/tensor_ggml.hpp
@@ -3,6 +3,7 @@
 
 #include <array>
 #include <cstring>
+#include <filesystem>
 #include <fstream>
 #include <stdexcept>
 #include <string>
@@ -105,7 +106,7 @@ namespace sd {
 
     template <typename T>
     inline Tensor<T> load_tensor_from_file_as_tensor(const std::string& file_path) {
-        std::ifstream file(file_path, std::ios::binary);
+        std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
         if (!file.is_open()) {
             throw std::runtime_error("failed to open tensor file: " + file_path);
         }
diff --git a/src/core/util.cpp b/src/core/util.cpp
index b90f70ed3f..c1d463e4da 100644
--- a/src/core/util.cpp
+++ b/src/core/util.cpp
@@ -7,6 +7,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <exception>
+#include <filesystem>
 #include <fstream>
 #include <locale>
 #include <regex>
@@ -62,6 +63,21 @@ void replace_all_chars(std::string& str, char target, char replacement) {
     }
 }
 
+std::string escape_newlines(const std::string& text) {
+    std::string escaped;
+    escaped.reserve(text.size());
+    for (char ch : text) {
+        if (ch == '\n') {
+            escaped += "\\n";
+        } else if (ch == '\r') {
+            escaped += "\\r";
+        } else {
+            escaped += ch;
+        }
+    }
+    return escaped;
+}
+
 static std::string sd_vformat(const char* fmt, va_list ap) {
     char small[128];
     va_list ap2;
@@ -108,12 +124,12 @@ int round_up_to(int value, int base) {
 #include <windows.h>
 
 bool file_exists(const std::string& filename) {
-    DWORD attributes = GetFileAttributesA(filename.c_str());
+    DWORD attributes = GetFileAttributesW(std::filesystem::u8path(filename).c_str());
     return (attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY));
 }
 
 bool is_directory(const std::string& path) {
-    DWORD attributes = GetFileAttributesA(path.c_str());
+    DWORD attributes = GetFileAttributesW(std::filesystem::u8path(path).c_str());
     return (attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY));
 }
 
@@ -137,8 +153,8 @@ std::unique_ptr<MmapWrapper> MmapWrapper::create(const std::string& filename, bo
     void* mapped_data = nullptr;
     size_t file_size  = 0;
 
-    HANDLE file_handle = CreateFileA(
-        filename.c_str(),
+    HANDLE file_handle = CreateFileW(
+        std::filesystem::u8path(filename).c_str(),
         GENERIC_READ,
         FILE_SHARE_READ,
         nullptr,
@@ -360,12 +376,13 @@ int32_t sd_get_num_physical_cores() {
 static sd_progress_cb_t sd_progress_cb = nullptr;
 void* sd_progress_cb_data              = nullptr;
 
-static sd_preview_cb_t sd_preview_cb = nullptr;
-static void* sd_preview_cb_data      = nullptr;
-preview_t sd_preview_mode            = PREVIEW_NONE;
-int sd_preview_interval              = 1;
-bool sd_preview_denoised             = true;
-bool sd_preview_noisy                = false;
+static sd_preview_info_t sd_preview_info = {};
+static sd_preview_cb_t sd_preview_cb     = nullptr;
+static void* sd_preview_cb_data          = nullptr;
+preview_t sd_preview_mode                = PREVIEW_NONE;
+int sd_preview_interval                  = 1;
+bool sd_preview_denoised                 = true;
+bool sd_preview_noisy                    = false;
 
 static sd_graph_eval_callback_t sd_backend_eval_cb = nullptr;
 static void* sd_backend_eval_cb_data               = nullptr;
@@ -644,9 +661,14 @@ void* sd_log_cb_data         = nullptr;
 static void sd_log_dispatch(sd_log_level_t level, const std::string& origin, const std::string& text) {
     if (sd_log_cb == nullptr)
         return;
-    std::string message = origin + " - " + text;
-    if (message.back() != '\n') {
-        message += '\n';
+    std::string message = text;
+    while (!message.empty() && (message.back() == '\n' || message.back() == '\r')) {
+        message.pop_back();
+    }
+    if (origin == "ggml") {
+        message = origin + ": " + message + '\n';
+    } else {
+        message += " --- " + origin + '\n';
     }
     sd_log_cb(level, message.c_str(), sd_log_cb_data);
 }
@@ -656,7 +678,7 @@ void log_printf(sd_log_level_t level, const char* file, int line, const char* fo
     va_start(args, format);
     std::string message = sd_vformat(format, args);
     va_end(args);
-    std::string origin = sd_format("%s:%-4d", sd_basename(file).c_str(), line);
+    std::string origin = sd_format("%s:%d", sd_basename(file).c_str(), line);
     sd_log_dispatch(level, origin, message);
 }
 
@@ -691,6 +713,7 @@ void sd_set_progress_callback(sd_progress_cb_t cb, void* data) {
     sd_progress_cb_data = data;
 }
 void sd_set_preview_callback(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data) {
+    sd_preview_info     = {};
     sd_preview_cb       = cb;
     sd_preview_cb_data  = data;
     sd_preview_mode     = mode;
@@ -704,6 +727,17 @@ void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data) {
     sd_backend_eval_cb_data = data;
 }
 
+void sd_begin_preview_pass(int total_steps) {
+    if (sd_preview_cb != nullptr) {
+        ++sd_preview_info.sample_pass;
+        sd_preview_info.total_steps = total_steps;
+    }
+}
+
+sd_preview_info_t sd_get_preview_info() {
+    return sd_preview_info;
+}
+
 sd_preview_cb_t sd_get_preview_callback() {
     return sd_preview_cb;
 }
diff --git a/src/core/util.h b/src/core/util.h
index 334224da26..0929f60e01 100644
--- a/src/core/util.h
+++ b/src/core/util.h
@@ -29,6 +29,7 @@ bool contains(const std::string& str, const std::string& substr);
 std::string sd_format(const char* fmt, ...);
 
 void replace_all_chars(std::string& str, char target, char replacement);
+std::string escape_newlines(const std::string& text);
 
 int round_up_to(int value, int base);
 
@@ -101,6 +102,7 @@ std::vector<std::pair<std::string, float>> split_quotation_attention(
 sd_progress_cb_t sd_get_progress_callback();
 void* sd_get_progress_callback_data();
 
+void sd_begin_preview_pass(int total_steps);
 sd_preview_cb_t sd_get_preview_callback();
 void* sd_get_preview_callback_data();
 preview_t sd_get_preview_mode();
diff --git a/src/detailer.cpp b/src/detailer.cpp
index bbf9c6880e..72d01dc6f0 100644
--- a/src/detailer.cpp
+++ b/src/detailer.cpp
@@ -970,6 +970,7 @@ bool adetail_image(adetailer_ctx_t* context,
         generation.pm_params           = {};
         generation.pulid_params        = {};
         generation.hires.enabled       = false;
+        generation.image_preprocess    = {};
         if (params.steps > 0) {
             generation.sample_params.sample_steps        = params.steps;
             generation.sample_params.custom_sigmas       = nullptr;
diff --git a/src/device_residency_manager.h b/src/device_residency_manager.h
index 0a2a8895f3..0be0d2ee42 100644
--- a/src/device_residency_manager.h
+++ b/src/device_residency_manager.h
@@ -27,6 +27,8 @@ struct DeviceMemoryRequest {
     size_t pending_allocation_bytes = 0;
     size_t runtime_resident_bytes   = 0;
     size_t max_backend_bytes        = 0;
+    // Repeated identical compute: a check that allocates nothing may reuse this owner's last free-memory reading.
+    bool reuse_device_query = false;
 
     // Runtime buffers only; the manager accounts for weights separately.
     size_t runtime_peak_bytes() const {
diff --git a/src/model.h b/src/model.h
index 3fdf1a63ed..0c1a37a66c 100644
--- a/src/model.h
+++ b/src/model.h
@@ -62,6 +62,12 @@ enum SDVersion {
     VERSION_SENSENOVA_U1_5,
     VERSION_LLADA_IMAGE,
     VERSION_ESRGAN,
+    VERSION_PIXART,
+    VERSION_MING_IMAGE,
+    VERSION_Z_IMAGE_L2P,
+    VERSION_IRIS,
+    VERSION_FLUX3,
+    VERSION_MOSS_IMAGE,
     VERSION_COUNT,
 };
 
@@ -114,6 +120,10 @@ static inline bool sd_version_is_flux(SDVersion version) {
     return false;
 }
 
+static inline bool sd_version_is_flux3(SDVersion version) {
+    return version == VERSION_FLUX3;
+}
+
 static inline bool sd_version_is_flux2(SDVersion version) {
     if (version == VERSION_FLUX2 || version == VERSION_FLUX2_KLEIN) {
         return true;
@@ -174,6 +184,10 @@ static inline bool sd_version_is_z_image(SDVersion version) {
     return false;
 }
 
+static inline bool sd_version_is_z_image_l2p(SDVersion version) {
+    return version == VERSION_Z_IMAGE_L2P;
+}
+
 static inline bool sd_version_is_llada_image(SDVersion version) {
     if (version == VERSION_LLADA_IMAGE) {
         return true;
@@ -252,6 +266,18 @@ static inline bool sd_version_is_sensenova_u1(SDVersion version) {
     return version == VERSION_SENSENOVA_U1_5;
 }
 
+static inline bool sd_version_is_pixart(SDVersion version) {
+    return version == VERSION_PIXART;
+}
+
+static inline bool sd_version_supports_video_generation(SDVersion version) {
+    return version == VERSION_SVD || sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_lingbot_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version);
+}
+
+static inline bool sd_version_supports_image_generation(SDVersion version) {
+    return !sd_version_supports_video_generation(version);
+}
+
 static inline bool sd_version_uses_flux_vae(SDVersion version) {
     if (sd_version_is_flux(version) || sd_version_is_z_image(version) || sd_version_is_boogu_image(version) || sd_version_is_longcat(version)) {
         return true;
@@ -260,14 +286,21 @@ static inline bool sd_version_uses_flux_vae(SDVersion version) {
 }
 
 static inline bool sd_version_uses_flux2_vae(SDVersion version) {
-    if (sd_version_is_flux2(version) || sd_version_is_ernie_image(version) || sd_version_is_lens(version) || sd_version_is_ideogram4(version) || sd_version_is_sefi_image(version) || sd_version_is_llada_image(version)) {
+    if (sd_version_is_flux2(version) ||
+        sd_version_is_flux3(version) ||
+        sd_version_is_ernie_image(version) ||
+        sd_version_is_lens(version) ||
+        sd_version_is_ideogram4(version) ||
+        sd_version_is_sefi_image(version) ||
+        sd_version_is_llada_image(version) ||
+        version == VERSION_MOSS_IMAGE) {
         return true;
     }
     return false;
 }
 
 static inline bool sd_version_uses_wan_vae(SDVersion version) {
-    if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_qwen_image(version) || sd_version_is_krea2(version) || sd_version_is_anima(version)) {
+    if (sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_qwen_image(version) || sd_version_is_krea2(version) || sd_version_is_anima(version) || version == VERSION_MING_IMAGE) {
         return true;
     }
     return false;
@@ -291,6 +324,7 @@ static inline bool sd_version_is_inpaint(SDVersion version) {
 static inline bool sd_version_is_dit(SDVersion version) {
     if (sd_version_is_flux(version) ||
         sd_version_is_flux2(version) ||
+        sd_version_is_flux3(version) ||
         sd_version_is_ltxav(version) ||
         sd_version_is_minimax_h3(version) ||
         sd_version_is_sd3(version) ||
@@ -301,6 +335,8 @@ static inline bool sd_version_is_dit(SDVersion version) {
         version == VERSION_HIDREAM_O1 ||
         sd_version_is_anima(version) ||
         sd_version_is_z_image(version) ||
+        sd_version_is_z_image_l2p(version) ||
+        version == VERSION_MING_IMAGE ||
         sd_version_is_llada_image(version) ||
         sd_version_is_boogu_image(version) ||
         sd_version_is_ernie_image(version) ||
@@ -312,7 +348,10 @@ static inline bool sd_version_is_dit(SDVersion version) {
         sd_version_is_sefi_image(version) ||
         sd_version_is_krea2(version) ||
         sd_version_is_mage_flow(version) ||
-        sd_version_is_sensenova_u1(version)) {
+        sd_version_is_sensenova_u1(version) ||
+        sd_version_is_pixart(version) ||
+        version == VERSION_IRIS ||
+        version == VERSION_MOSS_IMAGE) {
         return true;
     }
     return false;
@@ -326,6 +365,29 @@ static inline bool sd_version_is_control(SDVersion version) {
     return version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2;
 }
 
+// These model families consume uploaded reference images as generation
+// conditioning. Inpaint and control variants keep their init-image semantics.
+static inline bool sd_version_supports_ref_image_conditioning(SDVersion version) {
+    if (sd_version_is_inpaint(version) || sd_version_is_control(version) || version == VERSION_CHROMA_RADIANCE) {
+        return false;
+    }
+
+    return sd_version_is_unet_edit(version) ||
+           sd_version_is_flux(version) ||
+           sd_version_is_flux3(version) ||
+           sd_version_is_flux2(version) ||
+           sd_version_is_qwen_image(version) ||
+           sd_version_is_longcat(version) ||
+           sd_version_is_z_image(version) ||
+           sd_version_is_boogu_image(version) ||
+           sd_version_is_llada_image(version) ||
+           sd_version_is_krea2(version) ||
+           sd_version_is_mage_flow(version) ||
+           sd_version_is_anima(version) ||
+           sd_version_is_sefi_image(version) ||
+           sd_version_is_pid(version);
+}
+
 static bool sd_version_is_inpaint_or_unet_edit(SDVersion version) {
     return sd_version_is_unet_edit(version) || sd_version_is_inpaint(version) || sd_version_is_control(version);
 }
diff --git a/src/model/adapter/lora.hpp b/src/model/adapter/lora.hpp
index cb368feade..1e5377ccc8 100644
--- a/src/model/adapter/lora.hpp
+++ b/src/model/adapter/lora.hpp
@@ -1,6 +1,7 @@
 #ifndef __SD_MODEL_ADAPTER_LORA_HPP__
 #define __SD_MODEL_ADAPTER_LORA_HPP__
 
+#include <cmath>
 #include <mutex>
 #include "core/ggml_extend.h"
 #include "core/ggml_extend_backend.h"
@@ -11,15 +12,18 @@
 #include "model/adapter/lora_ops.h"
 #include "model_loader.h"
 #include "model_manager.h"
+#include "name_conversion.h"
 
 #define LORA_GRAPH_BASE_SIZE 10240
 
 struct LoraModel : public GGMLRunner {
     std::string lora_id;
-    float multiplier = 1.0f;
+    float multiplier     = 1.0f;
+    float metadata_alpha = -1.f;
     std::unordered_map<std::string, ggml_tensor*> lora_tensors;
     std::map<ggml_tensor*, ggml_tensor*> original_tensor_to_final_tensor;
     std::set<std::string> applied_lora_tensors;
+    std::set<std::string> swapped_lora_up_tensors;
     std::set<std::string> skipped_incompatible_lora_tensors;
     std::set<std::string> warned_incompatible_model_tensors;
     std::string file_path;
@@ -62,6 +66,31 @@ struct LoraModel : public GGMLRunner {
         if (load_failed || !registered_params.empty())
             return false;
         model_manager->set_n_threads(n_threads);
+        metadata_alpha      = -1.f;
+        const auto metadata = model_manager->loader().file_metadata(source_file);
+        const auto alpha    = metadata.find("alpha");
+        if (alpha != metadata.end()) {
+            try {
+                size_t parsed  = 0;
+                metadata_alpha = std::stof(alpha->second, &parsed);
+                if (parsed != alpha->second.size() || !std::isfinite(metadata_alpha) || metadata_alpha < 0.f) {
+                    throw std::invalid_argument("invalid alpha");
+                }
+            } catch (const std::exception&) {
+                LOG_ERROR("invalid LoRA alpha metadata '%s' in '%s'", alpha->second.c_str(), file_path.c_str());
+                return false;
+            }
+            LOG_VERBOSE("LoRA '%s' uses metadata alpha %.6g when tensor alpha/scale is absent", file_path.c_str(), metadata_alpha);
+        }
+        if (sd_version_is_minimax_h3(source_version)) {
+            for (const auto& entry : model_manager->loader().file_tensors(source_file)) {
+                const auto name = convert_tensor_name(entry.first, VERSION_COUNT);
+                if (ends_with(name, ".ff.net.0.proj.weight.lora_up")) {
+                    // Diffusers SwiGLU stores [value, gate]; MiniMax-H3 uses [gate, value].
+                    swapped_lora_up_tensors.insert(convert_tensor_name(entry.first, source_version));
+                }
+            }
+        }
         const auto sources = model_manager->loader().file_tensors(source_file, source_version);
         std::map<std::string, ggml_tensor*> tensors;
         std::map<std::string, ggml_tensor*> scalars;
@@ -116,6 +145,8 @@ struct LoraModel : public GGMLRunner {
         scalar_values.clear();
         original_tensor_to_final_tensor.clear();
         applied_lora_tensors.clear();
+        swapped_lora_up_tensors.clear();
+        metadata_alpha = -1.f;
         skipped_incompatible_lora_tensors.clear();
         warned_incompatible_model_tensors.clear();
         applied             = false;
@@ -163,6 +194,27 @@ struct LoraModel : public GGMLRunner {
 
             lora_tensors = std::move(new_lora_tensors);
         }
+
+        std::unordered_map<std::string, ggml_tensor*> new_lora_tensors;
+        for (const auto& [old_name, tensor] : lora_tensors) {
+            std::string new_name = old_name;
+            if (starts_with(old_name, "lora.model.diffusion_model.transformer_blocks.")) {
+                // Qwen Image 2.1 stores the gate before the projection in fused MLP weights.
+                for (const auto& suffix : {std::string(".img_mlp.gate_layer.weight."), std::string(".img_mlp.proj.weight.")}) {
+                    size_t pos = old_name.find(suffix);
+                    if (pos == std::string::npos) {
+                        continue;
+                    }
+                    std::string fused_name = old_name.substr(5, pos - 5) + ".img_mlp.gate_up.weight";
+                    if (model_tensor_names.find(fused_name) != model_tensor_names.end()) {
+                        new_name = "lora." + fused_name + (suffix == ".img_mlp.proj.weight." ? ".1." : ".") + old_name.substr(pos + suffix.size());
+                    }
+                    break;
+                }
+            }
+            new_lora_tensors[new_name] = tensor;
+        }
+        lora_tensors = std::move(new_lora_tensors);
     }
 
     ggml_tensor* get_lora_weight_diff(const std::string& model_tensor_name, ggml_context* ctx, ggml_backend_t backend) {
@@ -189,6 +241,10 @@ struct LoraModel : public GGMLRunner {
             auto iter = lora_tensors.find(lora_up_name);
             if (iter != lora_tensors.end()) {
                 lora_up = ggml_ext_cast_f32(ctx, backend, iter->second);
+                if (swapped_lora_up_tensors.count(lora_up_name)) {
+                    auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1);
+                    lora_up     = ggml_concat(ctx, halves[1], halves[0], 1);
+                }
             }
 
             iter = lora_tensors.find(lora_mid_name);
@@ -215,6 +271,7 @@ struct LoraModel : public GGMLRunner {
             float scale_value = 1.0f;
 
             int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
+            scale_value  = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f;
             iter         = lora_tensors.find(scale_name);
             if (iter != lora_tensors.end()) {
                 scale_value = scalar_value(iter->second);
@@ -702,6 +759,10 @@ struct LoraModel : public GGMLRunner {
             iter = lora_tensors.find(lora_up_name);
             if (iter != lora_tensors.end()) {
                 lora_up = iter->second;
+                if (swapped_lora_up_tensors.count(lora_up_name)) {
+                    auto halves = ggml_ext_chunk(ctx, lora_up, 2, 1);
+                    lora_up     = ggml_concat(ctx, halves[1], halves[0], 1);
+                }
                 if (is_conv2d && lora_up->type != GGML_TYPE_F16) {
                     lora_up = ggml_cast(ctx, lora_up, GGML_TYPE_F16);
                 }
@@ -770,6 +831,7 @@ struct LoraModel : public GGMLRunner {
             std::string scale_tensor_name;
 
             int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1];
+            scale_value  = metadata_alpha >= 0.f ? metadata_alpha / rank : 1.f;
             iter         = lora_tensors.find(scale_name);
             if (iter != lora_tensors.end()) {
                 scale_value       = scalar_value(iter->second);
diff --git a/src/model/adapter/lora_ops.cpp b/src/model/adapter/lora_ops.cpp
index 7dacfd164d..5a6f905de9 100644
--- a/src/model/adapter/lora_ops.cpp
+++ b/src/model/adapter/lora_ops.cpp
@@ -11,7 +11,9 @@ ggml_tensor* ggml_ext_merge_lora(ggml_context* ctx,
                                  ggml_tensor* lora_mid) {
     ggml_tensor* updown;
     // flat lora tensors to multiply it
-    int64_t lora_up_rows  = lora_up->ne[ggml_n_dims(lora_up) - 1];
+    // ggml_n_dims drops the output dimension when a linear layer has one output.
+    int lora_up_n_dims    = ggml_n_dims(lora_up);
+    int64_t lora_up_rows  = lora_up_n_dims == 1 ? 1 : lora_up->ne[lora_up_n_dims - 1];
     lora_up               = ggml_reshape_2d(ctx, lora_up, ggml_nelements(lora_up) / lora_up_rows, lora_up_rows);
     auto lora_down_n_dims = ggml_n_dims(lora_down);
     // assume n_dims should always be a multiple of 2 (otherwise rank 1 doesn't work)
diff --git a/src/model/common/ggml_block.hpp b/src/model/common/ggml_block.hpp
index 991b88f28b..be6dfea3d7 100644
--- a/src/model/common/ggml_block.hpp
+++ b/src/model/common/ggml_block.hpp
@@ -1,6 +1,7 @@
 #ifndef __SD_MODEL_COMMON_GGML_BLOCK_HPP__
 #define __SD_MODEL_COMMON_GGML_BLOCK_HPP__
 
+#include <cmath>
 #include <cstdint>
 #include <map>
 #include <memory>
@@ -12,6 +13,7 @@
 #include <vector>
 
 #include "core/ggml_extend.h"
+#include "core/ggml_extend_backend.h"
 #include "core/ggml_runner.h"
 #include "model.h"
 
@@ -161,6 +163,10 @@ class Linear : public UnaryBlock {
             wtype = GGML_TYPE_F32;
         }
         params["weight"] = ggml_new_tensor_2d(ctx, wtype, in_features, out_features);
+        if (ggml_is_quantized(wtype)) {
+            // forward() only ever multiplies by it, so on CPU it may live repacked.
+            sd_tag_matmul_only_weight(params["weight"]);
+        }
         if (bias) {
             enum ggml_type wtype = GGML_TYPE_F32;
             params["bias"]       = ggml_new_tensor_1d(ctx, wtype, out_features);
@@ -204,6 +210,35 @@ class Linear : public UnaryBlock {
         force_prec_f32 = force_prec_f32_;
     }
 
+    // For callers that fold this layer's input/output scaling into neighbouring fused ops: the
+    // weight when forward() is exactly ggml_ext_linear(x, weight, nullptr, prec_f32(),
+    // effective_scale()) (no bias, weight scale, adapter, INT8 or FP8 path), nullptr otherwise.
+    ggml_tensor* foldable_weight(GGMLRunnerContext* ctx) {
+        ggml_tensor* w = params["weight"];
+        if (bias || has_weight_scale || ctx->weight_adapter || w->type == GGML_TYPE_I8) {
+            return nullptr;
+        }
+#ifndef SD_USE_UPSTREAM_GGML
+        if (w->type == GGML_TYPE_F8_E4M3 || w->type == GGML_TYPE_F8_E5M2) {
+            return nullptr;
+        }
+#endif
+        return w;
+    }
+
+    float effective_scale(GGMLRunnerContext* ctx) const {
+        return ctx->linear_scale > 0.f ? ctx->linear_scale : scale;
+    }
+
+    bool prec_f32() const {
+        return force_prec_f32;
+    }
+
+    bool has_quantized_weight() const {
+        auto it = params.find("weight");
+        return it != params.end() && it->second != nullptr && ggml_is_quantized(it->second->type);
+    }
+
     ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
         ggml_tensor* w            = params["weight"];
         const float scale         = ctx->linear_scale > 0.f ? ctx->linear_scale : this->scale;
@@ -214,9 +249,9 @@ class Linear : public UnaryBlock {
             if (ctx->backend != nullptr) {
                 ggml_tensor* fp8_matmul = ggml_mul_mat(ctx->ggml_ctx, w, x);
                 if (force_prec_f32) {
-                    ggml_mul_mat_set_prec(fp8_matmul, GGML_PREC_F32);
+                    ggml_prec_set_acc(fp8_matmul, GGML_PREC_F32);
                 }
-                supports_fp8_matmul = ggml_backend_supports_op(ctx->backend, fp8_matmul);
+                supports_fp8_matmul = ggml_backend_supports_op(ctx->backend, fp8_matmul) && !sd_backend_is_cpu(ctx->backend);
             }
             if (!supports_fp8_matmul) {
                 w = ggml_cast(ctx->ggml_ctx, w, GGML_TYPE_BF16);
@@ -303,7 +338,16 @@ class Linear : public UnaryBlock {
             forward_params.op_type               = WeightAdapter::ForwardParams::op_type_t::OP_LINEAR;
             forward_params.linear.force_prec_f32 = force_prec_f32;
             forward_params.linear.scale          = scale;
-            out                                  = ctx->weight_adapter->forward_with_lora(ctx->ggml_ctx, ctx->backend, x, w, linear_bias, prefix, forward_params);
+            if (sd_backend_is_cpu(ctx->backend) && sd_weight_may_be_cpu_repacked(w)) {
+                // W may be repacked (MUL_MAT-only): add x dW^T to the output instead of folding into W.
+                if (linear_bias != nullptr) {
+                    linear_bias = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, linear_bias, prefix + "bias");
+                }
+                out = ggml_ext_linear(ctx->ggml_ctx, x, w, linear_bias, force_prec_f32, scale);
+                out = ctx->weight_adapter->add_lora_to_output(ctx->ggml_ctx, ctx->backend, x, w, out, prefix, forward_params);
+            } else {
+                out = ctx->weight_adapter->forward_with_lora(ctx->ggml_ctx, ctx->backend, x, w, linear_bias, prefix, forward_params);
+            }
         } else {
             out = ggml_ext_linear(ctx->ggml_ctx, x, w, linear_bias, force_prec_f32, scale);
         }
@@ -335,15 +379,75 @@ class Embedding : public UnaryBlock {
 protected:
     int64_t embedding_dim;
     int64_t num_embeddings;
+    bool is_int8_tensorwise     = false;
+    int int8_convrot_group_size = 0;
+
+    static void get_rows_i8(ggml_tensor* dst, int ith, int nth, void* userdata) {
+        const auto* embedding     = static_cast<const Embedding*>(userdata);
+        const int group_size      = embedding->int8_convrot_group_size;
+        const auto* weight        = dst->src[0];
+        const auto* input_ids     = static_cast<const int32_t*>(dst->src[1]->data);
+        const auto* scales        = static_cast<const float*>(dst->src[2]->data);
+        const bool scalar_scale   = ggml_nelements(dst->src[2]) == 1;
+        const float normalization = group_size > 0 ? 1.f / std::sqrt(static_cast<float>(group_size)) : 1.f;
+        for (int64_t row = ith; row < dst->ne[1]; row += nth) {
+            const int32_t token = input_ids[row];
+            GGML_ASSERT(token >= 0 && token < weight->ne[1]);
+            const auto* src   = reinterpret_cast<const int8_t*>(static_cast<const char*>(weight->data) + token * weight->nb[1]);
+            auto* out         = reinterpret_cast<float*>(static_cast<char*>(dst->data) + row * dst->nb[1]);
+            const float scale = scales[scalar_scale ? 0 : token] * normalization;
+            for (int64_t i = 0; i < weight->ne[0]; ++i) {
+                out[i] = static_cast<float>(src[i]) * scale;
+            }
+            // The regular Hadamard rotation is symmetric and its own inverse.
+            for (int stride = 1; stride < group_size; stride *= 4) {
+                for (int64_t base = 0; base < weight->ne[0]; base += 4 * stride) {
+                    for (int j = 0; j < stride; ++j) {
+                        float* values      = out + base + j;
+                        const float a      = values[0];
+                        const float b      = values[stride];
+                        const float c      = values[2 * stride];
+                        const float d      = values[3 * stride];
+                        values[0]          = a + b + c - d;
+                        values[stride]     = a + b - c + d;
+                        values[2 * stride] = a - b + c + d;
+                        values[3 * stride] = -a + b + c + d;
+                    }
+                }
+            }
+        }
+    }
+
     void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map, const std::string prefix = "") override {
-        enum ggml_type wtype = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
-        if (!support_get_rows(wtype)) {
+        auto weight_storage     = tensor_storage_map.find(prefix + "weight");
+        is_int8_tensorwise      = weight_storage != tensor_storage_map.end() && weight_storage->second.is_int8_tensorwise;
+        int8_convrot_group_size = 0;
+        enum ggml_type wtype    = get_type(prefix + "weight", tensor_storage_map, GGML_TYPE_F32);
+        if (is_int8_tensorwise) {
+            GGML_ASSERT(wtype == GGML_TYPE_I8);
+            auto scale_storage = tensor_storage_map.find(prefix + "weight_scale");
+            GGML_ASSERT(scale_storage != tensor_storage_map.end());
+            const int64_t scale_nelements = scale_storage->second.nelements();
+            GGML_ASSERT(scale_nelements == 1 || scale_nelements == num_embeddings);
+            params["weight_scale"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, scale_nelements);
+            if (weight_storage->second.int8_convrot) {
+                int8_convrot_group_size = weight_storage->second.int8_convrot_group_size;
+                int remainder           = int8_convrot_group_size;
+                while (remainder > 1 && remainder % 4 == 0) {
+                    remainder /= 4;
+                }
+                GGML_ASSERT(remainder == 1 && embedding_dim % int8_convrot_group_size == 0);
+            }
+        } else if (!support_get_rows(wtype)) {
             wtype = GGML_TYPE_F32;
         }
         params["weight"] = ggml_new_tensor_2d(ctx, wtype, embedding_dim, num_embeddings);
     }
 
     enum ggml_op param_usage_op(const std::string& name) const override {
+        if (is_int8_tensorwise) {
+            return GGML_OP_CUSTOM;
+        }
         return name == "weight" ? GGML_OP_GET_ROWS : GGML_OP_NONE;
     }
 
@@ -363,9 +467,17 @@ class Embedding : public UnaryBlock {
         int64_t n = input_ids->ne[1];
         input_ids = ggml_reshape_1d(ctx->ggml_ctx, input_ids, input_ids->ne[0] * input_ids->ne[1]);
 
-        input_ids      = ggml_reshape_3d(ctx->ggml_ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
-        auto embedding = ggml_get_rows(ctx->ggml_ctx, weight, input_ids);
-        embedding      = ggml_reshape_3d(ctx->ggml_ctx, embedding, embedding->ne[0], embedding->ne[1] / n, n);
+        ggml_tensor* embedding;
+        if (is_int8_tensorwise) {
+            GGML_ASSERT(input_ids->type == GGML_TYPE_I32);
+            ggml_tensor* args[] = {weight, input_ids, params["weight_scale"]};
+            embedding           = ggml_custom_4d(ctx->ggml_ctx, GGML_TYPE_F32, embedding_dim, input_ids->ne[0], 1, 1,
+                                                 args, 3, get_rows_i8, GGML_N_TASKS_MAX, this);
+        } else {
+            input_ids = ggml_reshape_3d(ctx->ggml_ctx, input_ids, input_ids->ne[0], 1, input_ids->ne[1]);
+            embedding = ggml_get_rows(ctx->ggml_ctx, weight, input_ids);
+        }
+        embedding = ggml_reshape_3d(ctx->ggml_ctx, embedding, embedding->ne[0], embedding->ne[1] / n, n);
 
         // [N, n_token, embedding_dim]
         return embedding;
diff --git a/src/model/common/rope.hpp b/src/model/common/rope.hpp
index e447849366..a0b342240d 100644
--- a/src/model/common/rope.hpp
+++ b/src/model/common/rope.hpp
@@ -5,8 +5,10 @@
 #include <cassert>
 #include <cmath>
 #include <set>
+#include <utility>
 #include <vector>
 #include "core/ggml_extend.h"
+#include "core/ggml_extend_backend.h"
 #include "core/ggml_runner.h"
 #include "core/util.h"
 
@@ -16,6 +18,45 @@ namespace Rope {
         ErnieImage,
     };
 
+    struct SpatialRegion {
+        size_t begin;
+        size_t count;
+        float height_period;
+        float width_period;
+        int height_axis = 1;
+        int width_axis  = 2;
+    };
+
+    struct PositionLayout {
+        // Token ranges are relative to one batch item.
+        std::vector<SpatialRegion> images;
+        size_t token_count = 0;
+
+        void append_tokens(size_t count) {
+            token_count += count;
+        }
+
+        void append_image(int height, int width, int frames = 1, float height_step = 1.f, float width_step = 1.f) {
+            size_t count = static_cast<size_t>(height) * width * frames;
+            images.push_back({token_count, count, height * height_step, width * width_step});
+            append_tokens(count);
+        }
+    };
+
+    struct Frequency {
+        size_t axis;
+        float omega;
+    };
+
+    struct Embedding {
+        std::vector<float> values;
+        std::vector<std::vector<float>> ids;
+        PositionLayout positions;
+        std::vector<Frequency> frequencies;
+        EmbedNDLayout layout = EmbedNDLayout::Matrix;
+        int batch_size       = 1;
+    };
+
     enum class RefIndexMode {
         FIXED,
         INCREASE,
@@ -56,40 +97,25 @@ namespace Rope {
         return flat_vec;
     }
 
-    __STATIC_INLINE__ std::vector<std::vector<float>> rope(const std::vector<float>& pos,
-                                                           int dim,
-                                                           float theta,
-                                                           const std::vector<int>& axis_wrap_dims = {}) {
+    __STATIC_INLINE__ std::vector<float> rope_frequencies(int dim, float theta) {
         assert(dim % 2 == 0);
-        int half_dim = dim / 2;
-
+        int half_dim             = dim / 2;
         std::vector<float> scale = linspace(0.f, (dim * 1.f - 2) / dim, half_dim);
-
         std::vector<float> omega(half_dim);
         for (int i = 0; i < half_dim; ++i) {
             omega[i] = 1.0f / ::powf(1.f * theta, scale[i]);
         }
+        return omega;
+    }
 
+    __STATIC_INLINE__ std::vector<std::vector<float>> rope(const std::vector<float>& pos,
+                                                           const std::vector<float>& omega) {
+        int half_dim    = static_cast<int>(omega.size());
         size_t pos_size = pos.size();
         std::vector<std::vector<float>> out(pos_size, std::vector<float>(half_dim));
         for (size_t i = 0; i < pos_size; ++i) {
             for (size_t j = 0; j < half_dim; ++j) {
                 float angle = pos[i] * omega[j];
-                if (!axis_wrap_dims.empty()) {
-                    size_t wrap_size = axis_wrap_dims.size();
-                    // mod batch size since we only store this for one item in the batch
-                    size_t wrap_idx = wrap_size > 0 ? (i % wrap_size) : 0;
-                    int wrap_dim    = axis_wrap_dims[wrap_idx];
-                    if (wrap_dim > 0) {
-                        constexpr float TWO_PI = 6.28318530717958647692f;
-                        float cycles           = omega[j] * wrap_dim / TWO_PI;
-                        // closest periodic harmonic, necessary to ensure things neatly tile
-                        // without this round, things don't tile at the boundaries and you end up
-                        // with the model knowing what is "center"
-                        float rounded = std::round(cycles);
-                        angle         = pos[i] * TWO_PI * rounded / wrap_dim;
-                    }
-                }
 
                 out[i][j] = angle;
             }
@@ -108,6 +134,12 @@ namespace Rope {
         return result;
     }
 
+    __STATIC_INLINE__ std::vector<std::vector<float>> rope(const std::vector<float>& pos,
+                                                           int dim,
+                                                           float theta) {
+        return rope(pos, rope_frequencies(dim, theta));
+    }
+
     // Generate IDs for image patches and text
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_flux_txt_ids(int bs, int context_len, int axes_dim_num, std::set<int> arange_dims) {
         auto txt_ids = std::vector<std::vector<float>>(bs * context_len, std::vector<float>(axes_dim_num, 0.0f));
@@ -136,12 +168,16 @@ namespace Rope {
                                                                        int patch_size,
                                                                        int bs,
                                                                        int axes_dim_num,
-                                                                       int index       = 0,
-                                                                       int h_offset    = 0,
-                                                                       int w_offset    = 0,
-                                                                       bool scale_rope = false) {
+                                                                       int index              = 0,
+                                                                       int h_offset           = 0,
+                                                                       int w_offset           = 0,
+                                                                       bool scale_rope        = false,
+                                                                       PositionLayout* layout = nullptr) {
         int h_len = (h + (patch_size / 2)) / patch_size;
         int w_len = (w + (patch_size / 2)) / patch_size;
+        if (layout) {
+            layout->append_image(h_len, w_len);
+        }
         std::vector<std::vector<float>> img_ids(h_len * w_len, std::vector<float>(axes_dim_num, 0.0));
 
         int h_start = h_offset;
@@ -192,8 +228,8 @@ namespace Rope {
                                                   int bs,
                                                   const std::vector<float>& axis_thetas,
                                                   const std::vector<int>& axes_dim,
-                                                  const std::vector<std::vector<int>>& wrap_dims = {},
-                                                  EmbedNDLayout layout                           = EmbedNDLayout::Matrix) {
+                                                  EmbedNDLayout layout                = EmbedNDLayout::Matrix,
+                                                  std::vector<Frequency>* frequencies = nullptr) {
         std::vector<std::vector<float>> trans_ids = transpose(ids);
         size_t pos_len                            = ids.size() / bs;
         size_t num_axes                           = axes_dim.size();
@@ -205,19 +241,25 @@ namespace Rope {
         for (int d : axes_dim)
             emb_dim += d / 2;
 
+        if (frequencies) {
+            frequencies->clear();
+            frequencies->reserve(emb_dim);
+        }
         std::vector<std::vector<float>> emb(bs * pos_len, std::vector<float>(emb_dim * 2 * 2, 0.0));
         size_t offset = 0;
         for (size_t i = 0; i < num_axes; ++i) {
-            std::vector<int> axis_wrap_dims;
-            if (!wrap_dims.empty() && i < (int)wrap_dims.size()) {
-                axis_wrap_dims = wrap_dims[i];
-            }
             float axis_theta = 10000.0f;
             if (!axis_thetas.empty()) {
                 axis_theta = axis_thetas[std::min(i, axis_thetas.size() - 1)];
             }
+            auto omega = rope_frequencies(axes_dim[i], axis_theta);
+            if (frequencies) {
+                for (float frequency : omega) {
+                    frequencies->push_back({i, frequency});
+                }
+            }
             std::vector<std::vector<float>> rope_emb =
-                rope(trans_ids[i], axes_dim[i], axis_theta, axis_wrap_dims);  // [bs*pos_len, axes_dim[i]/2 * 2 * 2]
+                rope(trans_ids[i], omega);  // [bs*pos_len, axes_dim[i]/2 * 2 * 2]
             for (int b = 0; b < bs; ++b) {
                 for (int j = 0; j < pos_len; ++j) {
                     for (int k = 0; k < rope_emb[0].size(); ++k) {
@@ -253,10 +295,10 @@ namespace Rope {
                                                   int bs,
                                                   float theta,
                                                   const std::vector<int>& axes_dim,
-                                                  const std::vector<std::vector<int>>& wrap_dims = {},
-                                                  EmbedNDLayout layout                           = EmbedNDLayout::Matrix) {
+                                                  EmbedNDLayout layout                = EmbedNDLayout::Matrix,
+                                                  std::vector<Frequency>* frequencies = nullptr) {
         std::vector<float> axis_thetas(axes_dim.size(), theta);
-        return embed_nd(ids, bs, axis_thetas, axes_dim, wrap_dims, layout);
+        return embed_nd(ids, bs, axis_thetas, axes_dim, layout, frequencies);
     }
 
     __STATIC_INLINE__ std::vector<float> embed_interleaved_mrope(const std::vector<std::vector<float>>& ids,
@@ -264,7 +306,7 @@ namespace Rope {
                                                                  float theta,
                                                                  int head_dim,
                                                                  const std::vector<int>& mrope_section,
-                                                                 const std::vector<std::vector<int>>& axis_wrap_dims = {}) {
+                                                                 std::vector<Frequency>* frequencies = nullptr) {
         GGML_ASSERT(bs > 0);
         GGML_ASSERT(head_dim % 2 == 0);
         GGML_ASSERT(mrope_section.size() >= 3);
@@ -273,20 +315,26 @@ namespace Rope {
         size_t pos_len                            = ids.size() / bs;
         int half_dim                              = head_dim / 2;
 
+        auto omega = rope_frequencies(head_dim, theta);
+        if (frequencies) {
+            frequencies->clear();
+            for (float frequency : omega) {
+                frequencies->push_back({0, frequency});
+            }
+        }
         std::vector<std::vector<std::vector<float>>> axis_embs;
         axis_embs.reserve(3);
         for (int axis = 0; axis < 3; ++axis) {
-            std::vector<int> axis_wrap;
-            if (axis < static_cast<int>(axis_wrap_dims.size())) {
-                axis_wrap = axis_wrap_dims[axis];
-            }
-            axis_embs.push_back(rope(trans_ids[axis], head_dim, theta, axis_wrap));
+            axis_embs.push_back(rope(trans_ids[axis], omega));
         }
 
         std::vector<std::vector<float>> emb = axis_embs[0];
         for (int axis = 1; axis < 3; ++axis) {
             int length = std::min<int>(mrope_section[axis] * 3, half_dim);
             for (int freq_idx = axis; freq_idx < length; freq_idx += 3) {
+                if (frequencies) {
+                    (*frequencies)[freq_idx].axis = axis;
+                }
                 for (size_t pos_idx = 0; pos_idx < bs * pos_len; ++pos_idx) {
                     for (int k = 0; k < 4; ++k) {
                         emb[pos_idx][4 * freq_idx + k] = axis_embs[axis][pos_idx][4 * freq_idx + k];
@@ -298,13 +346,13 @@ namespace Rope {
         return flatten(emb);
     }
 
-    __STATIC_INLINE__ std::vector<float> embed_2d_interleaved(int height,
-                                                              int width,
-                                                              int dim,
-                                                              float theta    = 10000.f,
-                                                              float scale    = 16.f,
-                                                              int ref_grid_h = 0,
-                                                              int ref_grid_w = 0) {
+    __STATIC_INLINE__ Embedding embed_2d_interleaved(int height,
+                                                     int width,
+                                                     int dim,
+                                                     float theta    = 10000.f,
+                                                     float scale    = 16.f,
+                                                     int ref_grid_h = 0,
+                                                     int ref_grid_w = 0) {
         assert(dim % 4 == 0);
         int half_dim      = dim / 2;
         int dim_axis      = dim / 2;
@@ -318,6 +366,10 @@ namespace Rope {
             w_ntk       = std::pow(static_cast<float>(width) / static_cast<float>(ref_grid_w), power);
         }
 
+        Embedding result;
+        result.positions.append_image(height, width, 1,
+                                      height > 1 ? scale / (height - 1) : 1.f,
+                                      width > 1 ? scale / (width - 1) : 1.f);
         std::vector<float> x_pos;
         std::vector<float> y_pos;
         x_pos.reserve(static_cast<size_t>(height) * width);
@@ -326,13 +378,20 @@ namespace Rope {
             float y = height == 1 ? 0.f : scale * static_cast<float>(iy) / static_cast<float>(height - 1);
             for (int ix = 0; ix < width; ++ix) {
                 float x = width == 1 ? 0.f : scale * static_cast<float>(ix) / static_cast<float>(width - 1);
+                result.ids.push_back({0.f, y, x});
                 x_pos.push_back(x);
                 y_pos.push_back(y);
             }
         }
 
-        auto x_emb = rope(x_pos, dim_axis, theta * w_ntk);
-        auto y_emb = rope(y_pos, dim_axis, theta * h_ntk);
+        auto x_freq = rope_frequencies(dim_axis, theta * w_ntk);
+        auto y_freq = rope_frequencies(dim_axis, theta * h_ntk);
+        auto x_emb  = rope(x_pos, x_freq);
+        auto y_emb  = rope(y_pos, y_freq);
+        for (int i = 0; i < axis_half_dim; ++i) {
+            result.frequencies.push_back({2, x_freq[i]});
+            result.frequencies.push_back({1, y_freq[i]});
+        }
 
         std::vector<float> out(static_cast<size_t>(height) * width * half_dim * 4);
         for (int pos = 0; pos < height * width; ++pos) {
@@ -348,7 +407,8 @@ namespace Rope {
                 }
             }
         }
-        return out;
+        result.values = std::move(out);
+        return result;
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_refs_ids(int patch_size,
@@ -359,7 +419,8 @@ namespace Rope {
                                                                    RefIndexMode ref_index_mode,
                                                                    float ref_index_scale,
                                                                    bool scale_rope,
-                                                                   int base_offset = 0) {
+                                                                   int base_offset        = 0,
+                                                                   PositionLayout* layout = nullptr) {
         std::vector<std::vector<float>> ids;
         int curr_h_offset = 0;
         int curr_w_offset = 0;
@@ -386,7 +447,8 @@ namespace Rope {
                                             static_cast<int>(index * ref_index_scale),
                                             h_offset + base_offset,
                                             w_offset + base_offset,
-                                            scale_rope);
+                                            scale_rope,
+                                            layout);
             ids          = concat_ids(ids, ref_ids, bs);
 
             if (ref_index_mode == RefIndexMode::INCREASE) {
@@ -409,88 +471,53 @@ namespace Rope {
                                                                    const std::vector<ggml_tensor*>& ref_latents,
                                                                    RefIndexMode ref_index_mode,
                                                                    float ref_index_scale,
-                                                                   bool is_longcat) {
+                                                                   bool is_longcat,
+                                                                   PositionLayout* layout = nullptr) {
+        if (layout) {
+            layout->append_tokens(context_len);
+        }
         int x_index = is_longcat ? 1 : 0;
 
         auto txt_ids = is_longcat ? gen_longcat_txt_ids(bs, context_len, axes_dim_num) : gen_flux_txt_ids(bs, context_len, axes_dim_num, txt_arange_dims);
         int offset   = is_longcat ? context_len : 0;
-        auto img_ids = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, x_index, offset, offset);
+        auto img_ids = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, x_index, offset, offset, false, layout);
 
         auto ids = concat_ids(txt_ids, img_ids, bs);
         if (ref_latents.size() > 0) {
-            auto refs_ids = gen_refs_ids(patch_size, bs, axes_dim_num, x_index + 1, ref_latents, ref_index_mode, ref_index_scale, false, offset);
+            auto refs_ids = gen_refs_ids(patch_size, bs, axes_dim_num, x_index + 1, ref_latents, ref_index_mode, ref_index_scale, false, offset, layout);
             ids           = concat_ids(ids, refs_ids, bs);
         }
         return ids;
     }
 
     // Generate flux positional embeddings
-    __STATIC_INLINE__ std::vector<float> gen_flux_pe(int h,
-                                                     int w,
-                                                     int patch_size,
-                                                     int bs,
-                                                     int context_len,
-                                                     std::set<int> txt_arange_dims,
-                                                     const std::vector<ggml_tensor*>& ref_latents,
-                                                     RefIndexMode ref_index_mode,
-                                                     float ref_index_scale,
-                                                     int theta,
-                                                     bool circular_h,
-                                                     bool circular_w,
-                                                     const std::vector<int>& axes_dim,
-                                                     bool is_longcat) {
-        std::vector<std::vector<float>> ids = gen_flux_ids(h,
-                                                           w,
-                                                           patch_size,
-                                                           bs,
-                                                           static_cast<int>(axes_dim.size()),
-                                                           context_len,
-                                                           txt_arange_dims,
-                                                           ref_latents,
-                                                           ref_index_mode,
-                                                           ref_index_scale,
-                                                           is_longcat);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int h_len = (h + (patch_size / 2)) / patch_size;
-            int w_len = (w + (patch_size / 2)) / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                size_t cursor           = context_len;  // text first
-                const size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][cursor + token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][cursor + token_i] = w_len;
-                    }
-                }
-                cursor += img_tokens;
-                // reference latents
-                for (ggml_tensor* ref : ref_latents) {
-                    if (ref == nullptr) {
-                        continue;
-                    }
-                    int ref_h         = static_cast<int>(ref->ne[1]);
-                    int ref_w         = static_cast<int>(ref->ne[0]);
-                    int ref_h_l       = (ref_h + (patch_size / 2)) / patch_size;
-                    int ref_w_l       = (ref_w + (patch_size / 2)) / patch_size;
-                    size_t ref_tokens = static_cast<size_t>(ref_h_l) * static_cast<size_t>(ref_w_l);
-                    for (size_t token_i = 0; token_i < ref_tokens; ++token_i) {
-                        if (circular_h) {
-                            wrap_dims[1][cursor + token_i] = ref_h_l;
-                        }
-                        if (circular_w) {
-                            wrap_dims[2][cursor + token_i] = ref_w_l;
-                        }
-                    }
-                    cursor += ref_tokens;
-                }
-            }
-        }
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    __STATIC_INLINE__ Embedding gen_flux_pe(int h,
+                                            int w,
+                                            int patch_size,
+                                            int bs,
+                                            int context_len,
+                                            std::set<int> txt_arange_dims,
+                                            const std::vector<ggml_tensor*>& ref_latents,
+                                            RefIndexMode ref_index_mode,
+                                            float ref_index_scale,
+                                            int theta,
+                                            const std::vector<int>& axes_dim,
+                                            bool is_longcat) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_flux_ids(h,
+                                         w,
+                                         patch_size,
+                                         bs,
+                                         static_cast<int>(axes_dim.size()),
+                                         context_len,
+                                         txt_arange_dims,
+                                         ref_latents,
+                                         ref_index_mode,
+                                         ref_index_scale,
+                                         is_longcat, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_vid_ids(int t,
@@ -500,14 +527,18 @@ namespace Rope {
                                                                   int ph,
                                                                   int pw,
                                                                   int bs,
-                                                                  int t_offset    = 0,
-                                                                  int h_offset    = 0,
-                                                                  int w_offset    = 0,
-                                                                  bool scale_rope = false) {
+                                                                  int t_offset           = 0,
+                                                                  int h_offset           = 0,
+                                                                  int w_offset           = 0,
+                                                                  bool scale_rope        = false,
+                                                                  PositionLayout* layout = nullptr) {
         int t_len = (t + (pt / 2)) / pt;
         int h_len = (h + (ph / 2)) / ph;
         int w_len = (w + (pw / 2)) / pw;
 
+        if (layout) {
+            layout->append_image(h_len, w_len, t_len);
+        }
         std::vector<std::vector<float>> vid_ids(t_len * h_len * w_len, std::vector<float>(3, 0.0));
 
         if (scale_rope) {
@@ -573,7 +604,11 @@ namespace Rope {
                                                                          int bs,
                                                                          int context_len,
                                                                          const std::vector<ggml_tensor*>& ref_latents,
-                                                                         RefIndexMode ref_index_mode) {
+                                                                         RefIndexMode ref_index_mode,
+                                                                         PositionLayout* layout = nullptr) {
+        if (layout) {
+            layout->append_tokens(context_len);
+        }
         int h_len        = (h + (patch_size / 2)) / patch_size;
         int w_len        = (w + (patch_size / 2)) / patch_size;
         int txt_id_start = std::max(h_len, w_len) / 2;
@@ -585,90 +620,49 @@ namespace Rope {
             }
         }
         int axes_dim_num = 3;
-        auto img_ids     = gen_vid_ids(t, h, w, 1, patch_size, patch_size, bs, 0, 0, 0, true);
+        auto img_ids     = gen_vid_ids(t, h, w, 1, patch_size, patch_size, bs, 0, 0, 0, true, layout);
         auto ids         = concat_ids(txt_ids_repeated, img_ids, bs);
         if (ref_latents.size() > 0) {
             int ref_start_index = ref_index_mode == RefIndexMode::DECREASE ? 0 : 1;
-            auto refs_ids       = gen_refs_ids(patch_size, bs, axes_dim_num, ref_start_index, ref_latents, ref_index_mode, 1.f, true);
+            auto refs_ids       = gen_refs_ids(patch_size, bs, axes_dim_num, ref_start_index, ref_latents, ref_index_mode, 1.f, true, 0, layout);
             ids                 = concat_ids(ids, refs_ids, bs);
         }
         return ids;
     }
 
     // Generate qwen_image positional embeddings
-    __STATIC_INLINE__ std::vector<float> gen_qwen_image_pe(int t,
-                                                           int h,
-                                                           int w,
-                                                           int patch_size,
-                                                           int bs,
-                                                           int context_len,
-                                                           const std::vector<ggml_tensor*>& ref_latents,
-                                                           RefIndexMode ref_index_mode,
-                                                           int theta,
-                                                           bool circular_h,
-                                                           bool circular_w,
-                                                           const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_qwen_image_ids(t, h, w, patch_size, bs, context_len, ref_latents, ref_index_mode);
-        std::vector<std::vector<int>> wrap_dims;
-        // This logic simply stores the (pad and patch_adjusted) sizes of images so we can make sure rope correctly tiles
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int pad_h = (patch_size - (h % patch_size)) % patch_size;
-            int pad_w = (patch_size - (w % patch_size)) % patch_size;
-            int h_len = (h + pad_h) / patch_size;
-            int w_len = (w + pad_w) / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                const size_t total_tokens = ids.size();
-                // Track per-token wrap lengths for the row/column axes so only spatial tokens become periodic.
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(total_tokens / bs, 0));
-                size_t cursor           = context_len;  // ignore text tokens
-                const size_t img_tokens = static_cast<size_t>(t) * static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][cursor + token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][cursor + token_i] = w_len;
-                    }
-                }
-                cursor += img_tokens;
-                // For each reference image, store wrap sizes as well
-                for (ggml_tensor* ref : ref_latents) {
-                    if (ref == nullptr) {
-                        continue;
-                    }
-                    int ref_h           = static_cast<int>(ref->ne[1]);
-                    int ref_w           = static_cast<int>(ref->ne[0]);
-                    int ref_pad_h       = (patch_size - (ref_h % patch_size)) % patch_size;
-                    int ref_pad_w       = (patch_size - (ref_w % patch_size)) % patch_size;
-                    int ref_h_len       = (ref_h + ref_pad_h) / patch_size;
-                    int ref_w_len       = (ref_w + ref_pad_w) / patch_size;
-                    size_t ref_n_tokens = static_cast<size_t>(ref_h_len) * static_cast<size_t>(ref_w_len);
-                    for (size_t token_i = 0; token_i < ref_n_tokens; ++token_i) {
-                        if (circular_h) {
-                            wrap_dims[1][cursor + token_i] = ref_h_len;
-                        }
-                        if (circular_w) {
-                            wrap_dims[2][cursor + token_i] = ref_w_len;
-                        }
-                    }
-                    cursor += ref_n_tokens;
-                }
-            }
-        }
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    __STATIC_INLINE__ Embedding gen_qwen_image_pe(int t,
+                                                  int h,
+                                                  int w,
+                                                  int patch_size,
+                                                  int bs,
+                                                  int context_len,
+                                                  const std::vector<ggml_tensor*>& ref_latents,
+                                                  RefIndexMode ref_index_mode,
+                                                  int theta,
+                                                  const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_qwen_image_ids(t, h, w, patch_size, bs, context_len, ref_latents, ref_index_mode, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_mage_flow_pe(int h,
-                                                          int w,
-                                                          int bs,
-                                                          int context_len,
-                                                          const std::vector<ggml_tensor*>& ref_latents,
-                                                          int theta,
-                                                          const std::vector<int>& axes_dim) {
+    __STATIC_INLINE__ Embedding gen_mage_flow_pe(int h,
+                                                 int w,
+                                                 int bs,
+                                                 int context_len,
+                                                 const std::vector<ggml_tensor*>& ref_latents,
+                                                 int theta,
+                                                 const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.positions.append_tokens(context_len);
         const int axes_dim_num = static_cast<int>(axes_dim.size());
-        auto make_image_ids    = [=](int image_h, int image_w, int image_index) {
+        auto make_image_ids    = [=, &result](int image_h, int image_w, int image_index) {
             std::vector<std::vector<float>> image_ids(static_cast<size_t>(bs) * image_h * image_w,
                                                          std::vector<float>(axes_dim_num, 0.f));
+            result.positions.append_image(image_h, image_w);
             int h_start = -(image_h - image_h / 2);
             int w_start = -(image_w - image_w / 2);
             for (int b = 0; b < bs; ++b) {
@@ -692,15 +686,18 @@ namespace Rope {
                                           static_cast<int>(i + 1));
             ids          = concat_ids(ids, ref_ids, bs);
         }
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim);
+        result.ids    = std::move(ids);
+        result.values = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_lens_ids(int h,
                                                                    int w,
                                                                    int bs,
                                                                    int context_len,
-                                                                   bool scale_rope = true) {
-        auto img_ids_repeated = gen_flux_img_ids(h, w, 1, bs, 3, 0, 0, 0, scale_rope);
+                                                                   bool scale_rope        = true,
+                                                                   PositionLayout* layout = nullptr) {
+        auto img_ids_repeated = gen_flux_img_ids(h, w, 1, bs, 3, 0, 0, 0, scale_rope, layout);
 
         int txt_id_start = scale_rope ? std::max(h / 2, w / 2) : 0;
         auto txt_ids     = linspace<float>(1.f * txt_id_start, 1.f * context_len + txt_id_start, context_len);
@@ -711,44 +708,37 @@ namespace Rope {
             }
         }
 
+        if (layout) {
+            layout->append_tokens(context_len);
+        }
         return concat_ids(img_ids_repeated, txt_ids_repeated, bs);
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_lens_pe(int h,
-                                                     int w,
-                                                     int bs,
-                                                     int context_len,
-                                                     int theta,
-                                                     bool circular_h,
-                                                     bool circular_w,
-                                                     const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_lens_ids(h, w, bs, context_len, true);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            size_t pos_len = ids.size() / bs;
-            wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-            const size_t img_tokens = static_cast<size_t>(h) * static_cast<size_t>(w);
-            for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                if (circular_h) {
-                    wrap_dims[1][token_i] = h;
-                }
-                if (circular_w) {
-                    wrap_dims[2][token_i] = w;
-                }
-            }
-        }
-
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    __STATIC_INLINE__ Embedding gen_lens_pe(int h,
+                                            int w,
+                                            int bs,
+                                            int context_len,
+                                            int theta,
+                                            const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_lens_ids(h, w, bs, context_len, true, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     __STATIC_INLINE__ std::vector<std::vector<float>> gen_ernie_image_ids(int h,
                                                                           int w,
                                                                           int patch_size,
                                                                           int bs,
-                                                                          int context_len) {
+                                                                          int context_len,
+                                                                          PositionLayout* layout = nullptr) {
         int h_len = h / patch_size;
         int w_len = w / patch_size;
 
+        if (layout) {
+            layout->append_image(h_len, w_len);
+        }
         std::vector<std::vector<float>> img_ids(h_len * w_len, std::vector<float>(3, 0.0f));
         std::vector<float> h_ids = linspace<float>(0.f, static_cast<float>(h_len - 1), h_len);
         std::vector<float> w_ids = linspace<float>(0.f, static_cast<float>(w_len - 1), w_len);
@@ -774,39 +764,25 @@ namespace Rope {
             }
         }
 
+        if (layout) {
+            layout->append_tokens(context_len);
+        }
         return concat_ids(img_ids_repeated, txt_ids, bs);
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_ernie_image_pe(int h,
-                                                            int w,
-                                                            int patch_size,
-                                                            int bs,
-                                                            int context_len,
-                                                            int theta,
-                                                            bool circular_h,
-                                                            bool circular_w,
-                                                            const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_ernie_image_ids(h, w, patch_size, bs, context_len);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int h_len = h / patch_size;
-            int w_len = w / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                const size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][token_i] = w_len;
-                    }
-                }
-            }
-        }
-
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims, EmbedNDLayout::ErnieImage);
+    __STATIC_INLINE__ Embedding gen_ernie_image_pe(int h,
+                                                   int w,
+                                                   int patch_size,
+                                                   int bs,
+                                                   int context_len,
+                                                   int theta,
+                                                   const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.layout     = EmbedNDLayout::ErnieImage;
+        result.ids        = gen_ernie_image_ids(h, w, patch_size, bs, context_len, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     // Generate wan positional embeddings
@@ -905,7 +881,8 @@ namespace Rope {
                                                                       int context_len,
                                                                       int seq_multi_of,
                                                                       const std::vector<ggml_tensor*>& ref_latents,
-                                                                      RefIndexMode ref_index_mode) {
+                                                                      RefIndexMode ref_index_mode,
+                                                                      PositionLayout* layout = nullptr) {
         SD_UNUSED(ref_index_mode);
         int padded_context_len = context_len + bound_mod(context_len, seq_multi_of);
         auto txt_ids           = std::vector<std::vector<float>>(bs * padded_context_len, std::vector<float>(3, 0.0f));
@@ -913,11 +890,17 @@ namespace Rope {
             txt_ids[i][0] = (i % padded_context_len) + 1.f;
         }
 
+        if (layout) {
+            layout->append_tokens(padded_context_len);
+        }
         int axes_dim_num = 3;
         int index        = padded_context_len + 1;
-        auto img_ids     = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, index);
+        auto img_ids     = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, index, 0, 0, false, layout);
 
         int img_pad_len = bound_mod(static_cast<int>(img_ids.size() / bs), seq_multi_of);
+        if (layout) {
+            layout->append_tokens(img_pad_len);
+        }
         if (img_pad_len > 0) {
             std::vector<std::vector<float>> img_pad_ids(bs * img_pad_len, std::vector<float>(3, 0.f));
             img_ids = concat_ids(img_ids, img_pad_ids, bs);
@@ -936,7 +919,8 @@ namespace Rope {
                                                                           int patch_size,
                                                                           int bs,
                                                                           int context_len,
-                                                                          int seq_multi_of) {
+                                                                          int seq_multi_of,
+                                                                          PositionLayout* layout = nullptr) {
         int context_pad_len    = bound_mod(context_len, seq_multi_of);
         int padded_context_len = context_len + context_pad_len;
         auto txt_ids           = std::vector<std::vector<float>>(bs * padded_context_len, std::vector<float>(3, 0.0f));
@@ -947,11 +931,17 @@ namespace Rope {
             }
         }
 
+        if (layout) {
+            layout->append_tokens(padded_context_len);
+        }
         int axes_dim_num = 3;
         int index        = padded_context_len + 1;
-        auto img_ids     = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, index);
+        auto img_ids     = gen_flux_img_ids(h, w, patch_size, bs, axes_dim_num, index, 0, 0, false, layout);
 
         int img_pad_len = bound_mod(static_cast<int>(img_ids.size() / bs), seq_multi_of);
+        if (layout) {
+            layout->append_tokens(img_pad_len);
+        }
         if (img_pad_len > 0) {
             std::vector<std::vector<float>> img_pad_ids(bs * img_pad_len, std::vector<float>(3, 0.f));
             img_ids = concat_ids(img_ids, img_pad_ids, bs);
@@ -968,7 +958,8 @@ namespace Rope {
                                                                                int patch_size,
                                                                                int context_len,
                                                                                int sigvq_len,
-                                                                               int seq_multi_of) {
+                                                                               int seq_multi_of,
+                                                                               PositionLayout* layout = nullptr) {
         const int context_pad    = bound_mod(context_len, seq_multi_of);
         const int padded_context = context_len + context_pad;
         const int h_len          = (h + (patch_size / 2)) / patch_size;
@@ -994,11 +985,17 @@ namespace Rope {
             cursor += 2;
         }
 
+        if (layout) {
+            layout->append_tokens(cap_ids.size());
+        }
         std::vector<std::vector<float>> img_ids;
         for (int copy = 0; copy < 2; ++copy) {
-            auto ids = gen_flux_img_ids(h, w, patch_size, 1, 3, cap_end_positions[copy]);
+            auto ids = gen_flux_img_ids(h, w, patch_size, 1, 3, cap_end_positions[copy], 0, 0, false, layout);
             img_ids.insert(img_ids.end(), ids.begin(), ids.end());
             img_ids.insert(img_ids.end(), image_pad, std::vector<float>(3, 0.f));
+            if (layout) {
+                layout->append_tokens(image_pad);
+            }
         }
 
         const int sigvq_start = static_cast<int>(cap_ids.size() + img_ids.size()) + 1;
@@ -1016,101 +1013,77 @@ namespace Rope {
         ids.insert(ids.end(), cap_ids.begin(), cap_ids.end());
         ids.insert(ids.end(), img_ids.begin(), img_ids.end());
         ids.insert(ids.end(), sigvq_ids.begin(), sigvq_ids.end());
+        if (layout) {
+            layout->append_tokens(sigvq_ids.size());
+        }
         SD_UNUSED(padded_image);
         return ids;
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_llada_image_edit_pe(int h,
-                                                                 int w,
-                                                                 int patch_size,
-                                                                 int context_len,
-                                                                 int sigvq_len,
-                                                                 int seq_multi_of,
-                                                                 int theta,
-                                                                 const std::vector<int>& axes_dim) {
-        auto ids = gen_llada_image_edit_ids(h, w, patch_size, context_len, sigvq_len, seq_multi_of);
-        return embed_nd(ids, 1, static_cast<float>(theta), axes_dim, {});
-    }
-
-    __STATIC_INLINE__ std::vector<float> gen_llada_image_pe(int h,
-                                                            int w,
-                                                            int patch_size,
-                                                            int bs,
-                                                            int context_len,
-                                                            int seq_multi_of,
-                                                            int theta,
-                                                            bool circular_h,
-                                                            bool circular_w,
-                                                            const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_llada_image_ids(h, w, patch_size, bs, context_len, seq_multi_of);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int pad_h = (patch_size - (h % patch_size)) % patch_size;
-            int pad_w = (patch_size - (w % patch_size)) % patch_size;
-            int h_len = (h + pad_h) / patch_size;
-            int w_len = (w + pad_w) / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                size_t cursor     = context_len + bound_mod(context_len, seq_multi_of);
-                size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][cursor + token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][cursor + token_i] = w_len;
-                    }
-                }
-            }
-        }
-
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
-    }
-
-    // Generate z_image positional embeddings
-    __STATIC_INLINE__ std::vector<float> gen_z_image_pe(int h,
+    __STATIC_INLINE__ Embedding gen_llada_image_edit_pe(int h,
                                                         int w,
                                                         int patch_size,
-                                                        int bs,
                                                         int context_len,
+                                                        int sigvq_len,
                                                         int seq_multi_of,
-                                                        const std::vector<ggml_tensor*>& ref_latents,
-                                                        RefIndexMode ref_index_mode,
                                                         int theta,
-                                                        bool circular_h,
-                                                        bool circular_w,
                                                         const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids = gen_z_image_ids(h, w, patch_size, bs, context_len, seq_multi_of, ref_latents, ref_index_mode);
-        std::vector<std::vector<int>> wrap_dims;
-        if ((circular_h || circular_w) && bs > 0 && axes_dim.size() >= 3) {
-            int pad_h = (patch_size - (h % patch_size)) % patch_size;
-            int pad_w = (patch_size - (w % patch_size)) % patch_size;
-            int h_len = (h + pad_h) / patch_size;
-            int w_len = (w + pad_w) / patch_size;
-            if (h_len > 0 && w_len > 0) {
-                size_t pos_len = ids.size() / bs;
-                wrap_dims.assign(axes_dim.size(), std::vector<int>(pos_len, 0));
-                size_t cursor     = context_len + bound_mod(context_len, seq_multi_of);  // skip text (and its padding)
-                size_t img_tokens = static_cast<size_t>(h_len) * static_cast<size_t>(w_len);
-                for (size_t token_i = 0; token_i < img_tokens; ++token_i) {
-                    if (circular_h) {
-                        wrap_dims[1][cursor + token_i] = h_len;
-                    }
-                    if (circular_w) {
-                        wrap_dims[2][cursor + token_i] = w_len;
-                    }
-                }
-            }
-        }
+        Embedding result;
+        result.batch_size = 1;
+        result.ids        = gen_llada_image_edit_ids(h, w, patch_size, context_len, sigvq_len, seq_multi_of, &result.positions);
+        result.values     = embed_nd(result.ids, 1, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
+    }
+
+    __STATIC_INLINE__ Embedding gen_llada_image_pe(int h,
+                                                   int w,
+                                                   int patch_size,
+                                                   int bs,
+                                                   int context_len,
+                                                   int seq_multi_of,
+                                                   int theta,
+                                                   const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_llada_image_ids(h, w, patch_size, bs, context_len, seq_multi_of, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
+    }
 
-        return embed_nd(ids, bs, static_cast<float>(theta), axes_dim, wrap_dims);
+    // Generate z_image positional embeddings
+    __STATIC_INLINE__ Embedding gen_z_image_pe(int h,
+                                               int w,
+                                               int patch_size,
+                                               int bs,
+                                               int context_len,
+                                               int seq_multi_of,
+                                               const std::vector<ggml_tensor*>& ref_latents,
+                                               RefIndexMode ref_index_mode,
+                                               int theta,
+                                               const std::vector<int>& axes_dim) {
+        Embedding result;
+        result.batch_size = bs;
+        result.ids        = gen_z_image_ids(h, w, patch_size, bs, context_len, seq_multi_of, ref_latents, ref_index_mode, &result.positions);
+        result.values     = embed_nd(result.ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
-    __STATIC_INLINE__ ggml_tensor* apply_rope(ggml_context* ctx,
+    __STATIC_INLINE__ ggml_tensor* apply_rope(GGMLRunnerContext* runner_ctx,
                                               ggml_tensor* x,
                                               ggml_tensor* pe,
                                               bool rope_interleaved = true) {
+        auto ctx = runner_ctx->ggml_ctx;
+#ifndef SD_USE_UPSTREAM_GGML
+        auto backend = runner_ctx->backend;
+        if (backend != nullptr && x->type == GGML_TYPE_F32 && pe->type == GGML_TYPE_F32 &&
+            x->ne[0] % 2 == 0 && pe->ne[0] == 2 && pe->ne[1] == 2 &&
+            pe->ne[2] == x->ne[0] / 2 && pe->ne[3] == x->ne[2]) {
+            auto out = ggml_rope_apply(ctx, x, pe, rope_interleaved);
+            if (ggml_backend_supports_op(backend, out)) {
+                return out;
+            }
+        }
+#endif
         // x: [N, L, n_head, d_head]
         // pe: [L, d_head/2, 2, 2], [[cos, -sin], [sin, cos]]
         int64_t d_head = x->ne[0];
@@ -1122,8 +1095,16 @@ namespace Rope {
             x = ggml_reshape_4d(ctx, x, 2, d_head / 2, L, n_head * N);  // [N * n_head, L, d_head/2, 2]
             x = ggml_cont(ctx, ggml_permute(ctx, x, 3, 0, 1, 2));       // [2, N * n_head, L, d_head/2]
         } else {
-            x = ggml_reshape_4d(ctx, x, d_head / 2, 2, L, n_head * N);       // [N * n_head, L, 2, d_head/2]
-            x = ggml_cont(ctx, ggml_ext_torch_permute(ctx, x, 0, 2, 3, 1));  // [2, N * n_head, L, d_head/2]
+            auto x_0 = ggml_view_3d(ctx, x, d_head / 2, L, n_head * N, x->nb[1], x->nb[2], 0);
+            auto x_1 = ggml_view_3d(ctx, x, d_head / 2, L, n_head * N, x->nb[1], x->nb[2], (d_head / 2) * x->nb[0]);
+
+            auto pe_2 = ggml_cont(ctx, ggml_permute(ctx, pe, 2, 3, 0, 1));
+            auto cos  = ggml_view_2d(ctx, pe_2, d_head / 2, L, pe_2->nb[1], 0);
+            auto sin  = ggml_view_2d(ctx, pe_2, d_head / 2, L, pe_2->nb[1], pe_2->nb[3]);
+
+            auto part_0 = ggml_sub(ctx, ggml_mul(ctx, x_0, cos), ggml_mul(ctx, x_1, sin));
+            auto part_1 = ggml_add(ctx, ggml_mul(ctx, x_0, sin), ggml_mul(ctx, x_1, cos));
+            return ggml_concat(ctx, part_0, part_1, 0);
         }
 
         int64_t offset = x->nb[2] * x->ne[2];
@@ -1141,13 +1122,78 @@ namespace Rope {
         auto pe_1 = ggml_view_3d(ctx, pe, pe->ne[0], pe->ne[1], pe->ne[2], pe->nb[1], pe->nb[2], offset * 1);  // [L, d_head/2, 2]
 
         auto x_out = ggml_add_inplace(ctx, ggml_mul(ctx, x_0, pe_0), ggml_mul(ctx, x_1, pe_1));  // [N * n_head, L, d_head/2, 2]
-        if (!rope_interleaved) {
-            x_out = ggml_cont(ctx, ggml_permute(ctx, x_out, 1, 0, 2, 3));  // [N * n_head, L, x, d_head/2]
-        }
-        x_out = ggml_reshape_3d(ctx, x_out, d_head, L, n_head * N);  // [N*n_head, L, d_head]
+        x_out      = ggml_reshape_3d(ctx, x_out, d_head, L, n_head * N);                         // [N*n_head, L, d_head]
         return x_out;
     }
 
+    __STATIC_INLINE__ bool fused_rope_attention_enabled() {
+        static const bool enabled = [] {
+            const char* e = getenv("SD_FUSED_ROPE_ATTN");
+            return e == nullptr || atoi(e) != 0;
+        }();
+        return enabled;
+    }
+
+    // Fuses RoPE, relayout, kv scale and F16 cast into ggml_rope_pe_permute; must match the unfused
+    // flash path bit for bit (same rounding order as ggml_rope_apply -> ggml_scale -> ggml_cpy).
+    // CUDA / HIP only; SD_FUSED_ROPE_ATTN=0 disables; returns nullptr when it does not apply.
+    __STATIC_INLINE__ ggml_tensor* attention_fused_rope(GGMLRunnerContext* ctx,
+                                                        ggml_tensor* q,
+                                                        ggml_tensor* k,
+                                                        ggml_tensor* v,
+                                                        ggml_tensor* pe,
+                                                        ggml_tensor* mask,
+                                                        float kv_scale,
+                                                        bool rope_interleaved) {
+#ifndef SD_GGML_FUSED_ROPE_ATTN
+        return nullptr;
+#else
+        auto backend = ctx->backend;
+        // same attn_scale override as ggml_ext_attention_ext: keeps large k/v finite in F16
+        if (ctx->attn_scale > 0.f) {
+            kv_scale = ctx->attn_scale;
+        }
+        // interleaved only: non-interleaved ggml_rope_apply reads 2 of 4 table entries, so it would differ
+        if (!rope_interleaved || !ctx->flash_attn_enabled || ctx->sage_attn_enabled || backend == nullptr ||
+            sd_backend_is_cpu(backend) || !fused_rope_attention_enabled()) {
+            return nullptr;
+        }
+        const int64_t d_head = q->ne[0];
+        const int64_t L      = q->ne[2];
+        const int64_t N      = q->ne[3];
+        // ggml_ext_attention_ext pads heads under 64 channels for the CUDA kernel; keep that path for them
+        if (d_head < 64 || d_head % 2 != 0 || k->ne[0] != d_head || v->ne[0] != d_head || k->ne[2] != L || v->ne[2] != L ||
+            k->ne[3] != N || v->ne[3] != N || v->ne[1] != k->ne[1] || q->ne[1] % k->ne[1] != 0) {
+            return nullptr;
+        }
+        if (q->type != GGML_TYPE_F32 || k->type != GGML_TYPE_F32 || v->type != GGML_TYPE_F32 ||
+            q->nb[0] != sizeof(float) || k->nb[0] != sizeof(float) || v->nb[0] != sizeof(float)) {
+            return nullptr;
+        }
+        if (pe == nullptr || pe->type != GGML_TYPE_F32 || !ggml_is_contiguous(pe) || pe->ne[0] != 2 || pe->ne[1] != 2 ||
+            pe->ne[2] != d_head / 2 || pe->ne[3] != L) {
+            return nullptr;
+        }
+        if (mask != nullptr && mask->ne[3] != 1) {
+            return nullptr;
+        }
+        const int n_rot = static_cast<int>(d_head);
+        auto gctx       = ctx->ggml_ctx;
+        auto q_hm       = ggml_rope_pe_permute_set_interleaved(ggml_rope_pe_permute(gctx, q, pe, n_rot, 1.f, GGML_TYPE_F32), true);
+        auto k_hm       = ggml_rope_pe_permute_set_interleaved(ggml_rope_pe_permute(gctx, k, pe, n_rot, kv_scale, GGML_TYPE_F16), true);
+        auto v_hm       = ggml_rope_pe_permute(gctx, v, nullptr, 0, kv_scale, GGML_TYPE_F16);
+        if (!ggml_backend_supports_op(backend, q_hm) || !ggml_backend_supports_op(backend, k_hm) ||
+            !ggml_backend_supports_op(backend, v_hm)) {
+            return nullptr;
+        }
+        // [d_head, L, heads, N] -> [d_head, L, heads * N], as the unfused path
+        q_hm = ggml_reshape_3d(gctx, q_hm, d_head, L, q->ne[1] * N);
+        k_hm = ggml_reshape_3d(gctx, k_hm, d_head, L, k->ne[1] * N);
+        v_hm = ggml_reshape_3d(gctx, v_hm, d_head, L, v->ne[1] * N);
+        return ggml_ext_attention_prepared(gctx, backend, q_hm, k_hm, v_hm, q->ne[1], N, kv_scale, mask);
+#endif
+    }
+
     __STATIC_INLINE__ ggml_tensor* attention(GGMLRunnerContext* ctx,
                                              ggml_tensor* q,
                                              ggml_tensor* k,
@@ -1159,10 +1205,13 @@ namespace Rope {
         // q,k,v: [N, L, n_head, d_head]
         // pe: [L, d_head/2, 2, 2]
         // return: [N, L, n_head*d_head]
+        if (auto fused = attention_fused_rope(ctx, q, k, v, pe, mask, kv_scale, rope_interleaved)) {
+            return fused;
+        }
         int64_t n_head = q->ne[1];
 
-        q = apply_rope(ctx->ggml_ctx, q, pe, rope_interleaved);  // [N*n_head, L, d_head]
-        k = apply_rope(ctx->ggml_ctx, k, pe, rope_interleaved);  // [N*n_head, L, d_head]
+        q = apply_rope(ctx, q, pe, rope_interleaved);  // [N*n_head, L, d_head]
+        k = apply_rope(ctx, k, pe, rope_interleaved);  // [N*n_head, L, d_head]
 
         auto x = ggml_ext_attention_ext(ctx, q, k, v, n_head, mask, true, ctx->flash_attn_enabled, kv_scale);  // [N, L, n_head*d_head]
         return x;
diff --git a/src/model/common/rope_circular.hpp b/src/model/common/rope_circular.hpp
new file mode 100644
index 0000000000..19dccb9593
--- /dev/null
+++ b/src/model/common/rope_circular.hpp
@@ -0,0 +1,65 @@
+#ifndef __SD_MODEL_COMMON_ROPE_CIRCULAR_HPP__
+#define __SD_MODEL_COMMON_ROPE_CIRCULAR_HPP__
+
+#include "model/common/rope.hpp"
+
+namespace Rope {
+    __STATIC_INLINE__ void apply_circular(Embedding& embedding, bool circular_x, bool circular_y) {
+        if (!circular_x && !circular_y) {
+            return;
+        }
+
+        GGML_ASSERT(embedding.batch_size > 0);
+        GGML_ASSERT(embedding.ids.size() % embedding.batch_size == 0);
+        size_t pos_len  = embedding.ids.size() / embedding.batch_size;
+        size_t half_dim = embedding.frequencies.size();
+        GGML_ASSERT(embedding.positions.token_count == pos_len);
+        GGML_ASSERT(embedding.values.size() == embedding.ids.size() * half_dim * 4);
+
+        constexpr float TWO_PI = 6.28318530717958647692f;
+        for (const auto& region : embedding.positions.images) {
+            GGML_ASSERT(region.begin <= pos_len && region.count <= pos_len - region.begin);
+            for (size_t j = 0; j < half_dim; ++j) {
+                const auto& frequency = embedding.frequencies[j];
+                float period          = 0.f;
+                if (circular_y && frequency.axis == static_cast<size_t>(region.height_axis)) {
+                    period = region.height_period;
+                } else if (circular_x && frequency.axis == static_cast<size_t>(region.width_axis)) {
+                    period = region.width_period;
+                }
+                if (period <= 0) {
+                    continue;
+                }
+
+                // Quantize to periodic harmonics while preserving the original coordinate offsets.
+                float rounded = std::round(frequency.omega * period / TWO_PI);
+                for (int b = 0; b < embedding.batch_size; ++b) {
+                    size_t begin = b * pos_len + region.begin;
+                    for (size_t i = begin; i < begin + region.count; ++i) {
+                        GGML_ASSERT(frequency.axis < embedding.ids[i].size());
+                        float angle   = embedding.ids[i][frequency.axis] * TWO_PI * rounded / period;
+                        float cos_val = std::cos(angle);
+                        float sin_val = std::sin(angle);
+                        if (embedding.layout == EmbedNDLayout::ErnieImage) {
+                            size_t cos_offset                = (i * half_dim + j) * 2;
+                            size_t sin_offset                = embedding.ids.size() * half_dim * 2 + cos_offset;
+                            embedding.values[cos_offset]     = cos_val;
+                            embedding.values[cos_offset + 1] = cos_val;
+                            embedding.values[sin_offset]     = sin_val;
+                            embedding.values[sin_offset + 1] = sin_val;
+                        } else {
+                            size_t offset                = (i * half_dim + j) * 4;
+                            embedding.values[offset]     = cos_val;
+                            embedding.values[offset + 1] = -sin_val;
+                            embedding.values[offset + 2] = sin_val;
+                            embedding.values[offset + 3] = cos_val;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+}  // namespace Rope
+
+#endif  // __SD_MODEL_COMMON_ROPE_CIRCULAR_HPP__
diff --git a/src/model/diffusion/anima.hpp b/src/model/diffusion/anima.hpp
index a8977c966f..a414d96e63 100644
--- a/src/model/diffusion/anima.hpp
+++ b/src/model/diffusion/anima.hpp
@@ -235,8 +235,8 @@ namespace Anima {
                 if (pe_k == nullptr) {
                     pe_k = pe_q;
                 }
-                auto q_rope = Rope::apply_rope(ctx->ggml_ctx, q4, pe_q, false);
-                auto k_rope = Rope::apply_rope(ctx->ggml_ctx, k4, pe_k, false);
+                auto q_rope = Rope::apply_rope(ctx, q4, pe_q, false);
+                auto k_rope = Rope::apply_rope(ctx, k4, pe_k, false);
                 attn_out    = ggml_ext_attention_ext(ctx,
                                                      q_rope,
                                                      k_rope,
@@ -603,34 +603,37 @@ namespace Anima {
             return std::pow(extrapolation_ratio, static_cast<float>(axis_dim) / static_cast<float>(axis_dim - 2));
         }
 
-        static std::vector<float> gen_anima_image_pe_vec(int bs,
-                                                         int h,
-                                                         int w,
-                                                         int patch_size,
-                                                         int theta,
-                                                         const std::vector<int>& axes_dim,
-                                                         float h_extrapolation_ratio,
-                                                         float w_extrapolation_ratio,
-                                                         float t_extrapolation_ratio,
-                                                         const std::vector<ggml_tensor*>& ref_latents) {
-            auto ids = Rope::gen_flux_ids(h,
-                                          w,
-                                          patch_size,
-                                          bs,
-                                          static_cast<int>(axes_dim.size()),
-                                          0,
-                                          {},
-                                          ref_latents,
-                                          Rope::RefIndexMode::FIXED,
-                                          1.0f,
-                                          false);
+        static Rope::Embedding gen_anima_image_pe_vec(int bs,
+                                                      int h,
+                                                      int w,
+                                                      int patch_size,
+                                                      int theta,
+                                                      const std::vector<int>& axes_dim,
+                                                      float h_extrapolation_ratio,
+                                                      float w_extrapolation_ratio,
+                                                      float t_extrapolation_ratio,
+                                                      const std::vector<ggml_tensor*>& ref_latents) {
+            Rope::Embedding result;
+            result.batch_size = bs;
+            result.ids        = Rope::gen_flux_ids(h,
+                                                   w,
+                                                   patch_size,
+                                                   bs,
+                                                   static_cast<int>(axes_dim.size()),
+                                                   0,
+                                                   {},
+                                                   ref_latents,
+                                                   Rope::RefIndexMode::FIXED,
+                                                   1.0f,
+                                                   false, &result.positions);
 
             std::vector<float> axis_thetas = {
                 static_cast<float>(theta) * calc_ntk_factor(t_extrapolation_ratio, axes_dim[0]),
                 static_cast<float>(theta) * calc_ntk_factor(h_extrapolation_ratio, axes_dim[1]),
                 static_cast<float>(theta) * calc_ntk_factor(w_extrapolation_ratio, axes_dim[2]),
             };
-            return Rope::embed_nd(ids, bs, axis_thetas, axes_dim);
+            result.values = Rope::embed_nd(result.ids, bs, axis_thetas, axes_dim, result.layout, &result.frequencies);
+            return result;
         }
 
         ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
@@ -657,16 +660,16 @@ namespace Anima {
             int64_t h_pad = x->ne[1] + pad_h;
             int64_t w_pad = x->ne[0] + pad_w;
 
-            image_pe_vec          = gen_anima_image_pe_vec(1,
-                                                           static_cast<int>(h_pad),
-                                                           static_cast<int>(w_pad),
-                                                           static_cast<int>(config.patch_size),
-                                                           config.theta,
-                                                           config.axes_dim,
-                                                           4.0f,
-                                                           4.0f,
-                                                           1.0f,
-                                                           ref_latents);
+            image_pe_vec          = finish_rope_pe(gen_anima_image_pe_vec(1,
+                                                                          static_cast<int>(h_pad),
+                                                                          static_cast<int>(w_pad),
+                                                                          static_cast<int>(config.patch_size),
+                                                                          config.theta,
+                                                                          config.axes_dim,
+                                                                          4.0f,
+                                                                          4.0f,
+                                                                          1.0f,
+                                                                          ref_latents));
             int64_t image_pos_len = static_cast<int64_t>(image_pe_vec.size()) / (2 * 2 * (config.head_dim / 2));
             auto image_pe         = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, image_pos_len);
             set_backend_tensor_data(image_pe, image_pe_vec.data());
diff --git a/src/model/diffusion/boogu.hpp b/src/model/diffusion/boogu.hpp
index 8c91bb55d2..1ca1370984 100644
--- a/src/model/diffusion/boogu.hpp
+++ b/src/model/diffusion/boogu.hpp
@@ -720,15 +720,18 @@ namespace Boogu {
         }
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_boogu_pe(int h,
-                                                      int w,
-                                                      int patch_size,
-                                                      int bs,
-                                                      int context_len,
-                                                      const std::vector<ggml_tensor*>& ref_latents,
-                                                      int theta,
-                                                      const std::vector<int>& axes_dim) {
-        std::vector<std::vector<float>> ids;
+    __STATIC_INLINE__ Rope::Embedding gen_boogu_pe(int h,
+                                                   int w,
+                                                   int patch_size,
+                                                   int bs,
+                                                   int context_len,
+                                                   const std::vector<ggml_tensor*>& ref_latents,
+                                                   int theta,
+                                                   const std::vector<int>& axes_dim) {
+        Rope::Embedding result;
+        result.batch_size = bs;
+        result.positions.append_tokens(context_len);
+        auto& ids = result.ids;
         ids.reserve(static_cast<size_t>(bs) * context_len);
         for (int b = 0; b < bs; b++) {
             for (int i = 0; i < context_len; i++) {
@@ -741,15 +744,18 @@ namespace Boogu {
         for (ggml_tensor* ref : ref_latents) {
             int ref_h_tokens = patched_token_count(ref->ne[1], patch_size);
             int ref_w_tokens = patched_token_count(ref->ne[0], patch_size);
+            result.positions.append_image(ref_h_tokens, ref_w_tokens);
             append_spatial_ids(ids, bs, pe_shift, ref_h_tokens, ref_w_tokens);
             pe_shift += std::max(ref_h_tokens, ref_w_tokens);
         }
 
         int h_tokens = patched_token_count(h, patch_size);
         int w_tokens = patched_token_count(w, patch_size);
+        result.positions.append_image(h_tokens, w_tokens);
         append_spatial_ids(ids, bs, pe_shift, h_tokens, w_tokens);
 
-        return Rope::embed_nd(ids, bs, static_cast<float>(theta), axes_dim);
+        result.values = Rope::embed_nd(ids, bs, static_cast<float>(theta), axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     struct BooguImageRunner : public DiffusionModelRunner {
@@ -793,14 +799,14 @@ namespace Boogu {
                 ref_latents.push_back(make_input(ref_latent_tensor));
             }
 
-            pe_vec      = gen_boogu_pe(static_cast<int>(x->ne[1]),
-                                       static_cast<int>(x->ne[0]),
-                                       config.patch_size,
-                                       static_cast<int>(x->ne[3]),
-                                       static_cast<int>(context->ne[1]),
-                                       ref_latents,
-                                       config.theta,
-                                       config.axes_dim);
+            pe_vec      = finish_rope_pe(gen_boogu_pe(static_cast<int>(x->ne[1]),
+                                                      static_cast<int>(x->ne[0]),
+                                                      config.patch_size,
+                                                      static_cast<int>(x->ne[3]),
+                                                      static_cast<int>(context->ne[1]),
+                                                      ref_latents,
+                                                      config.theta,
+                                                      config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
diff --git a/src/model/diffusion/control.hpp b/src/model/diffusion/control.hpp
index 34387d1b71..3b08ed1500 100644
--- a/src/model/diffusion/control.hpp
+++ b/src/model/diffusion/control.hpp
@@ -376,7 +376,7 @@ struct ControlNet : public GGMLRunner {
             hint = make_input(hint_tensor);
         }
 
-        auto runner_ctx = get_context();
+        auto runner_ctx = get_context(gf);
 
         auto outs = control_net.forward(&runner_ctx,
                                         x,
@@ -389,8 +389,7 @@ struct ControlNet : public GGMLRunner {
         if (guided_hint_input == nullptr && !outs.empty()) {
             guided_hint_output_ggml = outs[0];
             ggml_set_output(guided_hint_output_ggml);
-            cache(guided_hint_cache_name(), guided_hint_output_ggml);
-            ggml_build_forward_expand(gf, guided_hint_output_ggml);
+            runner_ctx.persist_cache_tensor(guided_hint_cache_name(), guided_hint_output_ggml);
         }
 
         control_outputs_ggml.reserve(outs.size() > 0 ? outs.size() - 1 : 0);
diff --git a/src/model/diffusion/ernie_image.hpp b/src/model/diffusion/ernie_image.hpp
index 81bf142a07..b284cb83f7 100644
--- a/src/model/diffusion/ernie_image.hpp
+++ b/src/model/diffusion/ernie_image.hpp
@@ -415,15 +415,13 @@ namespace ErnieImage {
             GGML_ASSERT(!context_tensor.empty());
             ggml_tensor* context = make_input(context_tensor);
 
-            pe_vec      = Rope::gen_ernie_image_pe(static_cast<int>(x->ne[1]),
-                                                   static_cast<int>(x->ne[0]),
-                                                   config.patch_size,
-                                                   static_cast<int>(x->ne[3]),
-                                                   static_cast<int>(context->ne[1]),
-                                                   config.theta,
-                                                   circular_y_enabled,
-                                                   circular_x_enabled,
-                                                   config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_ernie_image_pe(static_cast<int>(x->ne[1]),
+                                                                  static_cast<int>(x->ne[0]),
+                                                                  config.patch_size,
+                                                                  static_cast<int>(x->ne[3]),
+                                                                  static_cast<int>(context->ne[1]),
+                                                                  config.theta,
+                                                                  config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, config.axes_dim_sum, 1, pos_len, 2);
             set_backend_tensor_data(pe, pe_vec.data());
diff --git a/src/model/diffusion/flux.hpp b/src/model/diffusion/flux.hpp
index 67e79d3d96..825dabe13d 100644
--- a/src/model/diffusion/flux.hpp
+++ b/src/model/diffusion/flux.hpp
@@ -1548,20 +1548,18 @@ namespace Flux {
             } else if (version == VERSION_OVIS_IMAGE) {
                 txt_arange_dims = {1, 2};
             }
-            pe_vec      = Rope::gen_flux_pe(static_cast<int>(x->ne[1]),
-                                            static_cast<int>(x->ne[0]),
-                                            config.patch_size,
-                                            static_cast<int>(x->ne[3]),
-                                            static_cast<int>(context->ne[1]),
-                                            txt_arange_dims,
-                                            ref_latents,
-                                            ref_index_mode,
-                                            config.ref_index_scale,
-                                            config.theta,
-                                            circular_y_enabled,
-                                            circular_x_enabled,
-                                            config.axes_dim,
-                                            sd_version_is_longcat(version));
+            pe_vec      = finish_rope_pe(Rope::gen_flux_pe(static_cast<int>(x->ne[1]),
+                                                           static_cast<int>(x->ne[0]),
+                                                           config.patch_size,
+                                                           static_cast<int>(x->ne[3]),
+                                                           static_cast<int>(context->ne[1]),
+                                                           txt_arange_dims,
+                                                           ref_latents,
+                                                           ref_index_mode,
+                                                           config.ref_index_scale,
+                                                           config.theta,
+                                                           config.axes_dim,
+                                                           sd_version_is_longcat(version)));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
diff --git a/src/model/diffusion/flux3.hpp b/src/model/diffusion/flux3.hpp
new file mode 100644
index 0000000000..952587b013
--- /dev/null
+++ b/src/model/diffusion/flux3.hpp
@@ -0,0 +1,256 @@
+#ifndef __SD_MODEL_DIFFUSION_FLUX3_HPP__
+#define __SD_MODEL_DIFFUSION_FLUX3_HPP__
+
+#include "model/diffusion/flux.hpp"
+
+namespace Flux3 {
+
+    struct Flux3Config {
+        int64_t hidden_size = 3072;
+        int64_t context_dim = 20480;
+        int64_t channels    = 128;
+        int64_t mlp_dim     = 9216;
+        int heads           = 24;
+        int depth           = 5;
+        int single_depth    = 28;
+        bool flow_map       = false;
+
+        static Flux3Config detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) {
+            Flux3Config config;
+            int depth = 0, single_depth = 0;
+            bool detected = false;
+            for (const auto& [name, tensor] : tensors) {
+                if (!starts_with(name, prefix + ".")) {
+                    continue;
+                }
+                std::string key = name.substr(prefix.size() + 1);
+                if (key == "txt_in.weight") {
+                    detected           = true;
+                    config.context_dim = tensor.ne[0];
+                    config.hidden_size = tensor.ne[1];
+                } else if (key == "emb_in.image.weight") {
+                    config.channels = tensor.ne[0];
+                } else if (key == "single_blocks.0.mlp_out.weight") {
+                    config.mlp_dim = tensor.ne[0];
+                } else if (key == "ref_time_in.in_layer.weight") {
+                    config.flow_map = true;
+                }
+                if (starts_with(key, "txt_mode_blocks.")) {
+                    depth = std::max(depth, atoi(key.substr(16).c_str()) + 1);
+                }
+                if (starts_with(key, "single_blocks.")) {
+                    single_depth = std::max(single_depth, atoi(key.substr(14).c_str()) + 1);
+                }
+            }
+            if (depth > 0)
+                config.depth = depth;
+            if (single_depth > 0)
+                config.single_depth = single_depth;
+            config.heads = static_cast<int>(config.hidden_size / 128);
+            if (detected) {
+                LOG_VERBOSE("flux3: hidden_size = %" PRId64 ", context_dim = %" PRId64 ", depth = %d, single_depth = %d, flow_map = %s",
+                            config.hidden_size, config.context_dim, config.depth, config.single_depth, config.flow_map ? "true" : "false");
+            }
+            return config;
+        }
+    };
+
+    static ggml_tensor* token_slice(ggml_context* ctx, ggml_tensor* x, int64_t start, int64_t length) {
+        return ggml_view_3d(ctx, x, x->ne[0], length, x->ne[2], x->nb[1], x->nb[2], start * x->nb[1]);
+    }
+
+    struct Block : public GGMLBlock {
+        int heads;
+
+        Block(const Flux3Config& config)
+            : heads(config.heads) {
+            for (const auto* name : {"q_proj", "k_proj", "v_proj", "attn_out"}) {
+                blocks[name] = std::make_shared<Linear>(config.hidden_size, config.hidden_size, false);
+            }
+            blocks["mlp_in"]  = std::make_shared<Linear>(config.hidden_size, 2 * config.mlp_dim, false);
+            blocks["mlp_out"] = std::make_shared<Linear>(config.mlp_dim, config.hidden_size, false);
+            blocks["norm"]    = std::make_shared<Flux::QKNorm>(config.hidden_size / heads);
+        }
+
+        ggml_tensor* linear(GGMLRunnerContext* ctx, const std::string& name, ggml_tensor* x) {
+            return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, x);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, const std::vector<Flux::ModulationOut>& mods, const std::vector<int64_t>& lengths) {
+            auto gc            = ctx->ggml_ctx;
+            ggml_tensor* x_mod = nullptr;
+            int64_t offset     = 0;
+            for (size_t i = 0; i < lengths.size(); ++i) {
+                auto part = token_slice(gc, x, offset, lengths[i]);
+                part      = Flux::modulate(gc, ggml_norm(gc, part, 1e-6f), mods[i].shift, mods[i].scale);
+                x_mod     = x_mod ? ggml_concat(gc, x_mod, part, 1) : part;
+                offset += lengths[i];
+            }
+            auto split = [&](ggml_tensor* v) {
+                return ggml_reshape_4d(gc, v, x->ne[0] / heads, heads, x->ne[1], x->ne[2]);
+            };
+            auto norm        = std::dynamic_pointer_cast<Flux::QKNorm>(blocks["norm"]);
+            auto q           = norm->query_norm(ctx, split(linear(ctx, "q_proj", x_mod)));
+            auto k           = norm->key_norm(ctx, split(linear(ctx, "k_proj", x_mod)));
+            auto v           = split(linear(ctx, "v_proj", x_mod));
+            auto attn        = Rope::attention(ctx, q, k, v, pe, nullptr);
+            auto mlp         = ggml_ext_silu_act(gc, linear(ctx, "mlp_in", x_mod));
+            auto update      = ggml_add(gc, linear(ctx, "attn_out", attn), linear(ctx, "mlp_out", mlp));
+            ggml_tensor* out = nullptr;
+            offset           = 0;
+            for (size_t i = 0; i < lengths.size(); ++i) {
+                auto part = ggml_mul(gc, token_slice(gc, update, offset, lengths[i]), mods[i].gate);
+                part      = ggml_add(gc, token_slice(gc, x, offset, lengths[i]), part);
+                out       = out ? ggml_concat(gc, out, part, 1) : part;
+                offset += lengths[i];
+            }
+            return out;
+        }
+    };
+
+    struct Model : public GGMLBlock {
+        Flux3Config config;
+
+        Model(const Flux3Config& config)
+            : config(config) {
+            blocks["txt_in"]  = std::make_shared<Linear>(config.context_dim, config.hidden_size, false);
+            blocks["time_in"] = std::make_shared<Flux::MLPEmbedder>(256, config.hidden_size, false);
+            if (config.flow_map) {
+                blocks["ref_time_in"] = std::make_shared<Flux::MLPEmbedder>(256, config.hidden_size, false);
+            }
+            for (const std::string mode : {"image", "image_cond", "txt"}) {
+                blocks["early_stream_modulations." + mode]  = std::make_shared<Flux::Modulation>(config.hidden_size, false, false);
+                blocks["single_stream_modulations." + mode] = std::make_shared<Flux::Modulation>(config.hidden_size, false, false);
+                const std::string base                      = mode == "txt" ? "txt_mode_blocks." : "content_mode_blocks." + mode + ".";
+                for (int i = 0; i < config.depth; ++i) {
+                    blocks[base + std::to_string(i)] = std::make_shared<Block>(config);
+                }
+                if (mode != "txt") {
+                    blocks["emb_in." + mode] = std::make_shared<Linear>(config.channels, config.hidden_size, false);
+                }
+            }
+            for (int i = 0; i < config.single_depth; ++i) {
+                blocks["single_blocks." + std::to_string(i)] = std::make_shared<Block>(config);
+            }
+            blocks["final_layer.image"] = std::make_shared<Flux::LastLayer>(config.hidden_size, 1, config.channels, false, false);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image, ggml_tensor* text, ggml_tensor* reference, ggml_tensor* timestep, ggml_tensor* zero, ggml_tensor* target, ggml_tensor* pe) {
+            auto gc             = ctx->ggml_ctx;
+            const int64_t width = image->ne[0], height = image->ne[1];
+            auto time_in   = std::dynamic_pointer_cast<Flux::MLPEmbedder>(blocks["time_in"]);
+            auto image_vec = time_in->forward(ctx, ggml_ext_timestep_embedding(gc, timestep, 256, 10000, 1000.f));
+            auto clean_vec = time_in->forward(ctx, ggml_ext_timestep_embedding(gc, zero, 256, 10000, 1000.f));
+            if (config.flow_map) {
+                auto ref_time = std::dynamic_pointer_cast<Flux::MLPEmbedder>(blocks["ref_time_in"]);
+                auto ref_vec  = ref_time->forward(ctx, ggml_ext_timestep_embedding(gc, target, 256, 10000, 1000.f));
+                image_vec     = ggml_add(gc, image_vec, ref_vec);
+                clean_vec     = ggml_add(gc, clean_vec, ref_vec);
+            }
+            std::vector<std::string> modes{"image"};
+            std::vector<ggml_tensor*> inputs{DiT::patchify(gc, image, 1, 1)};
+            if (reference) {
+                modes.push_back("image_cond");
+                inputs.push_back(reference);
+            }
+            modes.push_back("txt");
+            inputs.push_back(text);
+            std::vector<Flux::ModulationOut> single_mods;
+            std::vector<int64_t> lengths;
+            ggml_tensor* joint = nullptr;
+            int64_t offset     = 0;
+            for (size_t m = 0; m < modes.size(); ++m) {
+                const auto& mode = modes[m];
+                auto vec         = m == 0 ? image_vec : clean_vec;
+                auto early       = std::dynamic_pointer_cast<Flux::Modulation>(blocks["early_stream_modulations." + mode])->forward(ctx, vec)[0];
+                single_mods.push_back(std::dynamic_pointer_cast<Flux::Modulation>(blocks["single_stream_modulations." + mode])->forward(ctx, vec)[0]);
+                lengths.push_back(inputs[m]->ne[1]);
+                auto h                 = std::dynamic_pointer_cast<Linear>(blocks[mode == "txt" ? "txt_in" : "emb_in." + mode])->forward(ctx, inputs[m]);
+                auto mode_pe           = ggml_view_4d(gc, pe, 2, 2, 64, lengths.back(), pe->nb[1], pe->nb[2], pe->nb[3], offset * pe->nb[3]);
+                const std::string base = mode == "txt" ? "txt_mode_blocks." : "content_mode_blocks." + mode + ".";
+                for (int i = 0; i < config.depth; ++i) {
+                    h = std::dynamic_pointer_cast<Block>(blocks[base + std::to_string(i)])->forward(ctx, h, mode_pe, {early}, {lengths.back()});
+                }
+                joint = joint ? ggml_concat(gc, joint, h, 1) : h;
+                offset += lengths.back();
+            }
+            for (int i = 0; i < config.single_depth; ++i) {
+                joint = std::dynamic_pointer_cast<Block>(blocks["single_blocks." + std::to_string(i)])->forward(ctx, joint, pe, single_mods, lengths);
+            }
+            auto out = std::dynamic_pointer_cast<Flux::LastLayer>(blocks["final_layer.image"])->forward(ctx, token_slice(gc, joint, 0, lengths[0]), image_vec);
+            return DiT::unpatchify(gc, out, height, width, 1, 1);
+        }
+    };
+
+    struct Flux3Runner : public DiffusionModelRunner {
+        Flux3Config config;
+        Model model;
+        std::vector<float> pe_values;
+        sd::Tensor<float> zero = sd::Tensor<float>::zeros({1});
+
+        Flux3Runner(ggml_backend_t backend, const String2TensorStorage& tensors, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : DiffusionModelRunner(backend, prefix, weight_manager),
+              config(Flux3Config::detect_from_weights(tensors, prefix)),
+              model(config) {
+            model.init(params_ctx, tensors, prefix);
+        }
+
+        std::string get_desc() override { return "flux3"; }
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        sd::Tensor<float> compute(int n_threads, const DiffusionParams& params) override {
+            if (params.ref_latents && params.ref_image_params.pass_to_dit && params.ref_latents->size() > 10) {
+                LOG_ERROR("FLUX.3 supports at most 10 reference images");
+                return {};
+            }
+            const auto* extra = diffusion_extra_as<Flux3DiffusionExtra>(params);
+            GGML_ASSERT(params.x && params.context && params.timesteps);
+            GGML_ASSERT(!config.flow_map || extra->target_timestep);
+            auto get_graph = [&]() {
+                auto gf   = new_graph_custom(FLUX_GRAPH_SIZE);
+                auto x    = make_input(*params.x);
+                auto text = make_input(*params.context);
+                GGML_ASSERT(x->ne[3] == 1 && text->ne[2] == 1);
+                Rope::Embedding embedding;
+                auto append_image = [&](int64_t w, int64_t h, float index) {
+                    embedding.positions.append_image(static_cast<int>(h), static_cast<int>(w), 1, 0.5f, 0.5f);
+                    for (int64_t y = 0; y < h; ++y) {
+                        for (int64_t x = 0; x < w; ++x) {
+                            embedding.ids.push_back({0.f, y * 0.5f, x * 0.5f, index});
+                        }
+                    }
+                };
+                append_image(x->ne[0], x->ne[1], 0.f);
+                ggml_tensor* references = nullptr;
+                if (params.ref_latents && params.ref_image_params.pass_to_dit) {
+                    int index = 0;
+                    for (const auto& ref : *params.ref_latents) {
+                        auto r = make_input(ref);
+                        append_image(r->ne[0], r->ne[1], -float(++index));
+                        r          = DiT::patchify(compute_ctx, r, 1, 1);
+                        references = references ? ggml_concat(compute_ctx, references, r, 1) : r;
+                    }
+                }
+                for (int64_t i = 0; i < text->ne[1]; ++i) {
+                    embedding.ids.push_back({0.f, 0.f, 0.f, static_cast<float>(i)});
+                }
+                embedding.positions.append_tokens(text->ne[1]);
+                embedding.values = Rope::embed_nd(embedding.ids, 1, 10000.f, {32, 32, 32, 32}, embedding.layout, &embedding.frequencies);
+                pe_values        = finish_rope_pe(std::move(embedding));
+                auto pe          = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, 64, pe_values.size() / 256);
+                set_backend_tensor_data(pe, pe_values.data());
+                auto runner_ctx = get_context();
+                auto out        = model.forward(&runner_ctx, x, text, references, make_input(*params.timesteps), make_input(zero),
+                                         config.flow_map ? make_input(*extra->target_timestep) : nullptr, pe);
+                ggml_build_forward_expand(gf, out);
+                return gf;
+            };
+            return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), params.x->dim());
+        }
+    };
+
+}  // namespace Flux3
+
+#endif  // __SD_MODEL_DIFFUSION_FLUX3_HPP__
diff --git a/src/model/diffusion/ideogram4.hpp b/src/model/diffusion/ideogram4.hpp
index 9e2aa36122..3254571db2 100644
--- a/src/model/diffusion/ideogram4.hpp
+++ b/src/model/diffusion/ideogram4.hpp
@@ -149,18 +149,21 @@ namespace Ideogram4 {
         return std::make_shared<Linear>(in_features, out_features, bias);
     }
 
-    __STATIC_INLINE__ std::vector<float> gen_ideogram4_pe(int grid_h,
-                                                          int grid_w,
-                                                          int bs,
-                                                          int context_len,
-                                                          int head_dim,
-                                                          int rope_theta,
-                                                          const std::vector<int>& mrope_section,
-                                                          bool circular_x = false,
-                                                          bool circular_y = false) {
+    __STATIC_INLINE__ Rope::Embedding gen_ideogram4_pe(int grid_h,
+                                                       int grid_w,
+                                                       int bs,
+                                                       int context_len,
+                                                       int head_dim,
+                                                       int rope_theta,
+                                                       const std::vector<int>& mrope_section) {
         GGML_ASSERT(bs == 1);
-        std::vector<std::vector<float>> ids(static_cast<size_t>(bs) * (context_len + grid_h * grid_w),
-                                            std::vector<float>(3, 0.f));
+        Rope::Embedding result;
+        result.batch_size = bs;
+        result.positions.append_tokens(context_len);
+        result.positions.append_image(grid_h, grid_w);
+        result.ids.assign(static_cast<size_t>(bs) * (context_len + grid_h * grid_w),
+                          std::vector<float>(3, 0.f));
+        auto& ids = result.ids;
 
         for (int i = 0; i < context_len; ++i) {
             ids[i] = {static_cast<float>(i), static_cast<float>(i), static_cast<float>(i)};
@@ -175,29 +178,13 @@ namespace Ideogram4 {
             }
         }
 
-        std::vector<std::vector<int>> axis_wrap_dims(3);
-        if (circular_y || circular_x) {
-            size_t total_len = static_cast<size_t>(bs) * (context_len + grid_h * grid_w);
-            axis_wrap_dims[1].assign(total_len, 0);
-            axis_wrap_dims[2].assign(total_len, 0);
-            if (circular_y) {
-                for (size_t idx = static_cast<size_t>(context_len); idx < total_len; ++idx) {
-                    axis_wrap_dims[1][idx] = grid_h;
-                }
-            }
-            if (circular_x) {
-                for (size_t idx = static_cast<size_t>(context_len); idx < total_len; ++idx) {
-                    axis_wrap_dims[2][idx] = grid_w;
-                }
-            }
-        }
-
-        return Rope::embed_interleaved_mrope(ids,
-                                             bs,
-                                             static_cast<float>(rope_theta),
-                                             head_dim,
-                                             mrope_section,
-                                             axis_wrap_dims);
+        result.values = Rope::embed_interleaved_mrope(ids,
+                                                      bs,
+                                                      static_cast<float>(rope_theta),
+                                                      head_dim,
+                                                      mrope_section,
+                                                      &result.frequencies);
+        return result;
     }
 
     class Ideogram4Attention : public GGMLBlock {
@@ -509,15 +496,13 @@ namespace Ideogram4 {
             int64_t head_dim = config.emb_dim / config.num_heads;
 
             auto runner_ctx = get_context();
-            pe_vec          = gen_ideogram4_pe(static_cast<int>(grid_h),
-                                               static_cast<int>(grid_w),
-                                               static_cast<int>(x->ne[3]),
-                                               static_cast<int>(context_len),
-                                               static_cast<int>(head_dim),
-                                               static_cast<int>(config.rope_theta),
-                                               config.mrope_section,
-                                               runner_ctx.circular_x_enabled,
-                                               runner_ctx.circular_y_enabled);
+            pe_vec          = finish_rope_pe(gen_ideogram4_pe(static_cast<int>(grid_h),
+                                                              static_cast<int>(grid_w),
+                                                              static_cast<int>(x->ne[3]),
+                                                              static_cast<int>(context_len),
+                                                              static_cast<int>(head_dim),
+                                                              static_cast<int>(config.rope_theta),
+                                                              config.mrope_section));
             auto pe         = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, head_dim / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
 
diff --git a/src/model/diffusion/iris.hpp b/src/model/diffusion/iris.hpp
new file mode 100644
index 0000000000..b90fbe9d44
--- /dev/null
+++ b/src/model/diffusion/iris.hpp
@@ -0,0 +1,450 @@
+#ifndef __SD_MODEL_DIFFUSION_IRIS_HPP__
+#define __SD_MODEL_DIFFUSION_IRIS_HPP__
+
+#include <array>
+#include <stdexcept>
+#include "model/diffusion/pid.hpp"
+
+namespace Iris {
+    struct IrisConfig {
+        int64_t hidden_size    = 2560;
+        int64_t depth          = 24;
+        int64_t dual_depth     = 8;
+        int64_t head_dim       = 128;
+        int64_t num_heads      = 20;
+        int64_t kv_heads       = 5;
+        int64_t mlp_dim        = 6826;
+        int64_t patch_size     = 16;
+        int64_t text_dim       = 2560;
+        int64_t text_len       = 300;
+        int64_t text_layers    = 12;
+        int64_t layer_heads    = 32;
+        int64_t layer_mlp_dim  = 3328;
+        int64_t pixel_dim      = 16;
+        int64_t pixel_attn_dim = 1280;
+        int64_t pixel_heads    = 10;
+        int64_t pixel_depth    = 4;
+
+        static IrisConfig detect_from_weights(const String2TensorStorage& weights, const std::string& prefix) {
+            IrisConfig config;
+            auto find = [&](const std::string& name) -> const TensorStorage& {
+                auto it = weights.find(prefix + "." + name);
+                if (it == weights.end()) {
+                    throw std::runtime_error("Iris-3B: missing weight " + prefix + "." + name);
+                }
+                return it->second;
+            };
+            config.hidden_size    = find("s_embedder.proj.weight").ne[1];
+            config.patch_size     = static_cast<int64_t>(std::sqrt(find("s_embedder.proj.weight").ne[0] / 3));
+            config.head_dim       = find("blocks.0.attn.q_norm_x.weight").ne[0];
+            config.num_heads      = config.hidden_size / config.head_dim;
+            config.kv_heads       = find("blocks.0.attn.k_proj_x.weight").ne[1] / config.head_dim;
+            config.mlp_dim        = find("blocks.0.mlp_x.w1.weight").ne[1];
+            config.text_dim       = find("y_embedder.refiner.proj.weight").ne[0];
+            config.text_len       = find("y_pos_embedding").ne[1];
+            config.text_layers    = find("y_embedder.layer_pool.weight").ne[0];
+            config.layer_mlp_dim  = find("y_embedder.layer_blocks.0.mlp.0.weight").ne[1];
+            config.pixel_dim      = find("pixel_embedder.proj.weight").ne[1];
+            config.pixel_attn_dim = find("pixel_blocks.0.compress.weight").ne[1];
+            config.pixel_heads    = config.pixel_attn_dim / find("pixel_blocks.0.attn.q_norm.weight").ne[0];
+            config.depth = config.dual_depth = config.pixel_depth = 0;
+            for (const auto& [name, tensor] : weights) {
+                if (!starts_with(name, prefix + ".")) {
+                    continue;
+                }
+                auto parts = split_string(name.substr(prefix.size() + 1), '.');
+                if (parts.size() > 2 && parts[0] == "blocks") {
+                    int64_t index = std::stoi(parts[1]);
+                    config.depth  = std::max(config.depth, index + 1);
+                    if (parts[2] == "adaln_img") {
+                        config.dual_depth = std::max(config.dual_depth, index + 1);
+                    }
+                } else if (parts.size() > 2 && parts[0] == "pixel_blocks") {
+                    config.pixel_depth = std::max(config.pixel_depth, int64_t(std::stoi(parts[1]) + 1));
+                }
+            }
+            if (config.text_layers != 12 || config.text_dim != 2560 || config.text_len != 300 ||
+                find("modulation_cores.adaln_img.weight").ne[1] != 6 * config.hidden_size ||
+                find("pixel_blocks.0.adaln.weight").ne[1] != 4 * config.pixel_dim * config.patch_size * config.patch_size) {
+                throw std::runtime_error("Iris-3B: unsupported checkpoint configuration");
+            }
+            LOG_VERBOSE("iris: hidden_size = %" PRId64 ", depth = %" PRId64 ", dual_depth = %" PRId64 ", heads = %" PRId64 "/%" PRId64 ", pixel_depth = %" PRId64,
+                        config.hidden_size, config.depth, config.dual_depth, config.num_heads, config.kv_heads, config.pixel_depth);
+            return config;
+        }
+    };
+
+    struct SelfAttention : public GGMLBlock {
+        int64_t dim;
+        int64_t heads;
+        bool qk_norm;
+
+        SelfAttention(int64_t dim, int64_t heads, bool qk_norm = true)
+            : dim(dim), heads(heads), qk_norm(qk_norm) {
+            blocks["qkv"]  = std::make_shared<Linear>(dim, 3 * dim, false);
+            blocks["proj"] = std::make_shared<Linear>(dim, dim, true);
+            if (qk_norm) {
+                blocks["q_norm"] = std::make_shared<RMSNorm>(dim / heads, 1e-6f);
+                blocks["k_norm"] = std::make_shared<RMSNorm>(dim / heads, 1e-6f);
+            }
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe = nullptr, ggml_tensor* mask = nullptr) {
+            auto c   = ctx->ggml_ctx;
+            auto qkv = split_qkv(c, std::dynamic_pointer_cast<Linear>(blocks["qkv"])->forward(ctx, x));
+            auto q   = ggml_reshape_4d(c, qkv[0], dim / heads, heads, x->ne[1], x->ne[2]);
+            auto k   = ggml_reshape_4d(c, qkv[1], dim / heads, heads, x->ne[1], x->ne[2]);
+            auto v   = ggml_reshape_4d(c, qkv[2], dim / heads, heads, x->ne[1], x->ne[2]);
+            if (qk_norm) {
+                q = std::dynamic_pointer_cast<RMSNorm>(blocks["q_norm"])->forward(ctx, q);
+                k = std::dynamic_pointer_cast<RMSNorm>(blocks["k_norm"])->forward(ctx, k);
+            }
+            if (pe) {
+                x = Rope::attention(ctx, q, k, v, pe, mask);
+            } else {
+                q = ggml_reshape_3d(c, q, dim, x->ne[1], x->ne[2]);
+                k = ggml_reshape_3d(c, k, dim, x->ne[1], x->ne[2]);
+                v = ggml_reshape_3d(c, v, dim, x->ne[1], x->ne[2]);
+                x = ggml_ext_attention_ext(ctx, q, k, v, heads, mask, false, ctx->flash_attn_enabled);
+            }
+            return std::dynamic_pointer_cast<Linear>(blocks["proj"])->forward(ctx, x);
+        }
+    };
+
+    struct TextBlock : public GGMLBlock {
+        bool layerwise;
+
+        TextBlock(int64_t dim, int64_t heads, int64_t mlp_dim, bool layerwise)
+            : layerwise(layerwise) {
+            blocks["norm1"] = std::make_shared<RMSNorm>(dim, 1e-6f);
+            blocks["norm2"] = std::make_shared<RMSNorm>(dim, 1e-6f);
+            blocks["attn"]  = std::make_shared<SelfAttention>(dim, heads, !layerwise);
+            if (layerwise) {
+                blocks["mlp.0"] = std::make_shared<Linear>(dim, mlp_dim, true);
+                blocks["mlp.2"] = std::make_shared<Linear>(mlp_dim, dim, true);
+            } else {
+                blocks["mlp"] = std::make_shared<Pid::FeedForward>(dim, mlp_dim);
+            }
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* mask = nullptr) {
+            auto h = std::dynamic_pointer_cast<RMSNorm>(blocks["norm1"])->forward(ctx, x);
+            h      = std::dynamic_pointer_cast<SelfAttention>(blocks["attn"])->forward(ctx, h, nullptr, mask);
+            x      = ggml_add(ctx->ggml_ctx, x, h);
+            h      = std::dynamic_pointer_cast<RMSNorm>(blocks["norm2"])->forward(ctx, x);
+            if (layerwise) {
+                h = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"])->forward(ctx, h);
+                h = ggml_silu(ctx->ggml_ctx, h);
+                h = std::dynamic_pointer_cast<Linear>(blocks["mlp.2"])->forward(ctx, h);
+            } else {
+                h = std::dynamic_pointer_cast<Pid::FeedForward>(blocks["mlp"])->forward(ctx, h);
+            }
+            return ggml_add(ctx->ggml_ctx, x, h);
+        }
+    };
+
+    struct TextEmbedder : public GGMLBlock {
+        IrisConfig config;
+
+        TextEmbedder(const IrisConfig& config)
+            : config(config) {
+            for (int i = 0; i < 2; ++i) {
+                blocks["layer_blocks." + std::to_string(i)]   = std::make_shared<TextBlock>(config.text_dim, config.layer_heads, config.layer_mlp_dim, true);
+                blocks["refiner.blocks." + std::to_string(i)] = std::make_shared<TextBlock>(config.hidden_size, config.num_heads, config.mlp_dim, false);
+            }
+            blocks["layer_pool"]   = std::make_shared<Linear>(config.text_layers, 1, true);
+            blocks["refiner.proj"] = std::make_shared<Linear>(config.text_dim, config.hidden_size, true);
+            blocks["refiner.norm"] = std::make_shared<RMSNorm>(config.hidden_size, 1e-6f);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* mask) {
+            auto c         = ctx->ggml_ctx;
+            int64_t tokens = x->ne[1];
+            int64_t batch  = x->ne[2];
+            // The encoder concatenates selected layers within each token's feature dimension.
+            x = ggml_reshape_3d(c, x, config.text_dim, config.text_layers, tokens * batch);
+            for (int i = 0; i < 2; ++i) {
+                x = std::dynamic_pointer_cast<TextBlock>(blocks["layer_blocks." + std::to_string(i)])->forward(ctx, x);
+            }
+            x = ggml_cont(c, ggml_permute(c, x, 1, 0, 2, 3));
+            x = std::dynamic_pointer_cast<Linear>(blocks["layer_pool"])->forward(ctx, x);
+            x = ggml_reshape_3d(c, x, config.text_dim, tokens, batch);
+            x = std::dynamic_pointer_cast<Linear>(blocks["refiner.proj"])->forward(ctx, x);
+            for (int i = 0; i < 2; ++i) {
+                x = std::dynamic_pointer_cast<TextBlock>(blocks["refiner.blocks." + std::to_string(i)])->forward(ctx, x, mask);
+            }
+            return std::dynamic_pointer_cast<RMSNorm>(blocks["refiner.norm"])->forward(ctx, x);
+        }
+    };
+
+    struct TrunkBlock : public GGMLBlock {
+        IrisConfig config;
+        bool dual;
+
+        TrunkBlock(const IrisConfig& config, bool dual)
+            : config(config), dual(dual) {
+            int64_t d = config.hidden_size;
+            for (const std::string stream : dual ? std::vector<std::string>{"x", "y"} : std::vector<std::string>{""}) {
+                std::string suffix                                = stream.empty() ? "" : "_" + stream;
+                std::string attn                                  = dual ? "attn." : "";
+                blocks[attn + "q_proj" + suffix]                  = std::make_shared<Linear>(d, d, false);
+                blocks[attn + "k_proj" + suffix]                  = std::make_shared<Linear>(d, config.kv_heads * config.head_dim, false);
+                blocks[attn + "v_proj" + suffix]                  = std::make_shared<Linear>(d, config.kv_heads * config.head_dim, false);
+                blocks[attn + "q_norm" + suffix]                  = std::make_shared<RMSNorm>(config.head_dim, 1e-6f);
+                blocks[attn + "k_norm" + suffix]                  = std::make_shared<RMSNorm>(config.head_dim, 1e-6f);
+                blocks[dual ? "attn.proj" + suffix : "attn_proj"] = std::make_shared<Linear>(d, d, true);
+                blocks["attn_gate" + suffix]                      = std::make_shared<Linear>(d, d, false);
+                blocks["norm" + suffix + "1"]                     = std::make_shared<RMSNorm>(d, 1e-6f);
+                blocks["norm" + suffix + "2"]                     = std::make_shared<RMSNorm>(d, 1e-6f);
+                blocks["attn_post_norm" + suffix]                 = std::make_shared<RMSNorm>(d, 1e-6f);
+                blocks["mlp_post_norm" + suffix]                  = std::make_shared<RMSNorm>(d, 1e-6f);
+                blocks["mlp" + suffix]                            = std::make_shared<Pid::FeedForward>(d, config.mlp_dim);
+            }
+        }
+
+        void init_params(ggml_context* ctx, const String2TensorStorage& = {}, const std::string = "") override {
+            if (dual) {
+                params["adaln_img.bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 6 * config.hidden_size);
+                params["adaln_txt.bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 6 * config.hidden_size);
+            } else {
+                params["adaln.bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 6 * config.hidden_size);
+            }
+        }
+
+        ggml_tensor* norm(GGMLRunnerContext* ctx, const std::string& name, ggml_tensor* x) {
+            return std::dynamic_pointer_cast<RMSNorm>(blocks[name])->forward(ctx, x);
+        }
+
+        ggml_tensor* linear(GGMLRunnerContext* ctx, const std::string& name, ggml_tensor* x) {
+            return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, x);
+        }
+
+        std::array<ggml_tensor*, 3> project(GGMLRunnerContext* ctx, ggml_tensor* h, const std::string& suffix) {
+            auto c        = ctx->ggml_ctx;
+            std::string a = dual ? "attn." : "";
+            auto q        = linear(ctx, a + "q_proj" + suffix, h);
+            auto k        = linear(ctx, a + "k_proj" + suffix, h);
+            auto v        = linear(ctx, a + "v_proj" + suffix, h);
+            q             = ggml_reshape_4d(c, q, config.head_dim, config.num_heads, h->ne[1], h->ne[2]);
+            k             = ggml_reshape_4d(c, k, config.head_dim, config.kv_heads, h->ne[1], h->ne[2]);
+            v             = ggml_reshape_4d(c, v, config.head_dim, config.kv_heads, h->ne[1], h->ne[2]);
+            return {norm(ctx, a + "q_norm" + suffix, q), norm(ctx, a + "k_norm" + suffix, k), v};
+        }
+
+        ggml_tensor* finish(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* h, ggml_tensor* out, const std::vector<ggml_tensor*>& mods, const std::string& suffix) {
+            auto c = ctx->ggml_ctx;
+            out    = ggml_mul(c, out, ggml_sigmoid(c, linear(ctx, "attn_gate" + suffix, h)));
+            out    = linear(ctx, dual ? "attn.proj" + suffix : "attn_proj", out);
+            out    = norm(ctx, "attn_post_norm" + suffix, out);
+            x      = ggml_add(c, x, ggml_mul(c, out, mods[2]));
+            h      = Pid::apply_adaln(c, norm(ctx, "norm" + suffix + "2", x), mods[3], mods[4]);
+            h      = std::dynamic_pointer_cast<Pid::FeedForward>(blocks["mlp" + suffix])->forward(ctx, h);
+            h      = norm(ctx, "mlp_post_norm" + suffix, h);
+            return ggml_add(c, x, ggml_mul(c, h, mods[5]));
+        }
+
+        std::pair<ggml_tensor*, ggml_tensor*> forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* y, ggml_tensor* img_mod, ggml_tensor* txt_mod, ggml_tensor* pe) {
+            auto c     = ctx->ggml_ctx;
+            int64_t nt = y->ne[1];
+            auto mx    = ggml_ext_chunk(c, ggml_add(c, img_mod, params[dual ? "adaln_img.bias" : "adaln.bias"]), 6, 0);
+            if (!dual) {
+                auto tokens = ggml_concat(c, y, x, 1);
+                auto h      = Pid::apply_adaln(c, norm(ctx, "norm1", tokens), mx[0], mx[1]);
+                auto qkv    = project(ctx, h, "");
+                auto out    = Rope::attention(ctx, qkv[0], qkv[1], qkv[2], pe, nullptr);
+                tokens      = finish(ctx, tokens, h, out, mx, "");
+                return {ggml_ext_slice(c, tokens, 1, nt, tokens->ne[1]), ggml_ext_slice(c, tokens, 1, 0, nt)};
+            }
+            auto my  = ggml_ext_chunk(c, ggml_add(c, txt_mod, params["adaln_txt.bias"]), 6, 0);
+            auto hx  = Pid::apply_adaln(c, norm(ctx, "norm_x1", x), mx[0], mx[1]);
+            auto hy  = Pid::apply_adaln(c, norm(ctx, "norm_y1", y), my[0], my[1]);
+            auto qx  = project(ctx, hx, "_x");
+            auto qy  = project(ctx, hy, "_y");
+            auto out = Rope::attention(ctx, ggml_concat(c, qy[0], qx[0], 2), ggml_concat(c, qy[1], qx[1], 2),
+                                       ggml_concat(c, qy[2], qx[2], 2), pe, nullptr);
+            x        = finish(ctx, x, hx, ggml_ext_slice(c, out, 1, nt, out->ne[1]), mx, "_x");
+            y        = finish(ctx, y, hy, ggml_ext_slice(c, out, 1, 0, nt), my, "_y");
+            return {x, y};
+        }
+    };
+
+    struct PixelBlock : public GGMLBlock {
+        IrisConfig config;
+
+        PixelBlock(const IrisConfig& config)
+            : config(config) {
+            auto d             = config.pixel_dim;
+            auto flat          = d * config.patch_size * config.patch_size;
+            blocks["norm1"]    = std::make_shared<RMSNorm>(d, 1e-6f);
+            blocks["norm2"]    = std::make_shared<RMSNorm>(d, 1e-6f);
+            blocks["compress"] = std::make_shared<Linear>(flat, config.pixel_attn_dim, true);
+            blocks["expand"]   = std::make_shared<Linear>(config.pixel_attn_dim, flat, true);
+            blocks["adaln"]    = std::make_shared<Linear>(config.hidden_size, 4 * flat, true);
+            blocks["attn"]     = std::make_shared<SelfAttention>(config.pixel_attn_dim, config.pixel_heads);
+            blocks["mlp.fc1"]  = std::make_shared<Linear>(d, d * 4, true);
+            blocks["mlp.fc2"]  = std::make_shared<Linear>(d * 4, d, true);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* cond, ggml_tensor* pe) {
+            auto c       = ctx->ggml_ctx;
+            auto d       = config.pixel_dim;
+            auto p2      = config.patch_size * config.patch_size;
+            auto patches = cond->ne[1];
+            auto batch   = cond->ne[2];
+            auto mods    = std::dynamic_pointer_cast<Linear>(blocks["adaln"])->forward(ctx, cond);
+            mods         = ggml_reshape_3d(c, mods, 4 * d, p2, patches * batch);
+            auto m       = ggml_ext_chunk(c, mods, 4, 0);
+            auto h       = std::dynamic_pointer_cast<RMSNorm>(blocks["norm1"])->forward(ctx, x);
+            h            = ggml_reshape_3d(c, h, d * p2, patches, batch);
+            h            = std::dynamic_pointer_cast<Linear>(blocks["compress"])->forward(ctx, h);
+            h            = std::dynamic_pointer_cast<SelfAttention>(blocks["attn"])->forward(ctx, h, pe);
+            h            = std::dynamic_pointer_cast<Linear>(blocks["expand"])->forward(ctx, h);
+            h            = ggml_reshape_3d(c, h, d, p2, patches * batch);
+            x            = ggml_add(c, x, Pid::apply_adaln(c, h, m[1], m[0]));
+            h            = std::dynamic_pointer_cast<RMSNorm>(blocks["norm2"])->forward(ctx, x);
+            h            = std::dynamic_pointer_cast<Linear>(blocks["mlp.fc1"])->forward(ctx, h);
+            h            = ggml_gelu_erf(c, h);
+            h            = std::dynamic_pointer_cast<Linear>(blocks["mlp.fc2"])->forward(ctx, h);
+            return ggml_add(c, x, Pid::apply_adaln(c, h, m[3], m[2]));
+        }
+    };
+
+    struct IrisModel : public GGMLBlock {
+        IrisConfig config;
+
+        IrisModel(const IrisConfig& config)
+            : config(config) {
+            blocks["s_embedder"]                 = std::make_shared<Pid::PatchTokenEmbedder>(3 * config.patch_size * config.patch_size, config.hidden_size);
+            blocks["t_embedder"]                 = std::make_shared<Pid::PixelDiTTimestepEmbedder>(config.hidden_size);
+            blocks["y_embedder"]                 = std::make_shared<TextEmbedder>(config);
+            blocks["modulation_cores.adaln_img"] = std::make_shared<Linear>(config.hidden_size, 6 * config.hidden_size, true);
+            blocks["modulation_cores.adaln_txt"] = std::make_shared<Linear>(config.hidden_size, 6 * config.hidden_size, true);
+            for (int64_t i = 0; i < config.depth; ++i) {
+                blocks["blocks." + std::to_string(i)] = std::make_shared<TrunkBlock>(config, i < config.dual_depth);
+            }
+            blocks["pixel_embedder"] = std::make_shared<Pid::PixelTokenEmbedder>(3, config.pixel_dim);
+            for (int64_t i = 0; i < config.pixel_depth; ++i) {
+                blocks["pixel_blocks." + std::to_string(i)] = std::make_shared<PixelBlock>(config);
+            }
+            blocks["final_layer"] = std::make_shared<Pid::FinalLayer>(config.pixel_dim, 3);
+        }
+
+        void init_params(ggml_context* ctx, const String2TensorStorage& = {}, const std::string = "") override {
+            params["y_pos_embedding"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, config.hidden_size, config.text_len, 1);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image, ggml_tensor* t, ggml_tensor* text, ggml_tensor* mask, ggml_tensor* joint_pe, ggml_tensor* pixel_pe, ggml_tensor* pixel_pos) {
+            auto c    = ctx->ggml_ctx;
+            int p     = static_cast<int>(config.patch_size);
+            auto x    = DiT::patchify(c, image, p, p, true);
+            x         = std::dynamic_pointer_cast<Pid::PatchTokenEmbedder>(blocks["s_embedder"])->forward(ctx, x);
+            auto te   = std::dynamic_pointer_cast<Pid::PixelDiTTimestepEmbedder>(blocks["t_embedder"])->forward(ctx, t);
+            te        = ggml_reshape_3d(c, te, config.hidden_size, 1, image->ne[3]);
+            auto cond = ggml_silu(c, te);
+            auto im   = std::dynamic_pointer_cast<Linear>(blocks["modulation_cores.adaln_img"])->forward(ctx, cond);
+            auto tm   = std::dynamic_pointer_cast<Linear>(blocks["modulation_cores.adaln_txt"])->forward(ctx, cond);
+            auto y    = std::dynamic_pointer_cast<TextEmbedder>(blocks["y_embedder"])->forward(ctx, text, mask);
+            y         = ggml_add(c, y, params["y_pos_embedding"]);
+            for (int64_t i = 0; i < config.depth; ++i) {
+                auto block     = std::dynamic_pointer_cast<TrunkBlock>(blocks["blocks." + std::to_string(i)]);
+                std::tie(x, y) = block->forward(ctx, x, y, im, tm, joint_pe);
+                sd::ggml_graph_cut::mark_graph_cut(x, "iris.blocks." + std::to_string(i), "x");
+                sd::ggml_graph_cut::mark_graph_cut(y, "iris.blocks." + std::to_string(i), "y");
+            }
+            cond = ggml_silu(c, ggml_add(c, x, te));
+            x    = std::dynamic_pointer_cast<Pid::PixelTokenEmbedder>(blocks["pixel_embedder"])->forward(ctx, image, p, pixel_pos);
+            for (int64_t i = 0; i < config.pixel_depth; ++i) {
+                x = std::dynamic_pointer_cast<PixelBlock>(blocks["pixel_blocks." + std::to_string(i)])->forward(ctx, x, cond, pixel_pe);
+                sd::ggml_graph_cut::mark_graph_cut(x, "iris.pixel_blocks." + std::to_string(i), "x");
+            }
+            x = std::dynamic_pointer_cast<Pid::FinalLayer>(blocks["final_layer"])->forward(ctx, x);
+            x = ggml_reshape_3d(c, x, 3 * p * p, cond->ne[1], image->ne[3]);
+            return DiT::unpatchify(c, x, image->ne[1] / p, image->ne[0] / p, p, p, false);
+        }
+    };
+
+    inline std::vector<float> image_rope(int64_t height, int64_t width, int64_t dim) {
+        std::vector<float> pe;
+        pe.reserve(height * width * dim * 2);
+        float step = 16.f / static_cast<float>(std::max<int64_t>(std::max(height, width) - 1, 1));
+        for (int64_t y = 0; y < height; ++y) {
+            for (int64_t x = 0; x < width; ++x) {
+                for (int64_t i = 0; i < dim / 4; ++i) {
+                    float frequency = std::pow(10000.f, -4.f * static_cast<float>(i) / static_cast<float>(dim));
+                    for (int64_t pos : {x, y}) {
+                        float angle = static_cast<float>(pos) * step * frequency;
+                        pe.insert(pe.end(), {std::cos(angle), -std::sin(angle), std::sin(angle), std::cos(angle)});
+                    }
+                }
+            }
+        }
+        return pe;
+    }
+
+    struct IrisRunner : public DiffusionModelRunner {
+        IrisConfig config;
+        IrisModel model;
+        std::vector<float> joint_pe_data;
+        std::vector<float> pixel_pe_data;
+        std::vector<float> pixel_pos_data;
+        std::vector<float> text_mask_data;
+
+        IrisRunner(ggml_backend_t backend, const String2TensorStorage& weights, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : DiffusionModelRunner(backend, prefix, weight_manager),
+              config(IrisConfig::detect_from_weights(weights, prefix)),
+              model(config) {
+            model.init(params_ctx, weights, prefix);
+        }
+
+        std::string get_desc() override { return "Iris-3B"; }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        sd::Tensor<float> compute(int n_threads, const DiffusionParams& inputs) override {
+            if (!inputs.x || !inputs.timesteps || !inputs.context || !inputs.y || inputs.y->empty()) {
+                LOG_ERROR("Iris-3B requires text features and their padding mask");
+                return {};
+            }
+            auto get_graph = [&]() {
+                auto gf   = new_graph_custom(196608);
+                auto x    = make_input(*inputs.x);
+                auto t    = make_input(*inputs.timesteps);
+                auto text = make_input(*inputs.context);
+                int h     = static_cast<int>(x->ne[1]);
+                int w     = static_cast<int>(x->ne[0]);
+                int nt    = static_cast<int>(config.text_len);
+                GGML_ASSERT(h % config.patch_size == 0 && w % config.patch_size == 0);
+                GGML_ASSERT(text->ne[0] == config.text_dim * config.text_layers && text->ne[1] == nt);
+                GGML_ASSERT(inputs.y->numel() == nt && x->ne[3] == 1);
+                joint_pe_data = Pid::make_rope_1d(nt, static_cast<int>(config.head_dim), 10000.f);
+                auto img      = image_rope(h / config.patch_size, w / config.patch_size, config.head_dim);
+                joint_pe_data.insert(joint_pe_data.end(), img.begin(), img.end());
+                pixel_pe_data  = image_rope(h / config.patch_size, w / config.patch_size, config.pixel_attn_dim / config.pixel_heads);
+                pixel_pos_data = Pid::make_pixel_abs_pos(h, w, static_cast<int>(config.pixel_dim));
+                text_mask_data.resize(nt * nt);
+                for (int q = 0; q < nt; ++q) {
+                    for (int k = 0; k < nt; ++k) {
+                        text_mask_data[q * nt + k] = inputs.y->values()[k] != 0.f || q == k ? 0.f : -INFINITY;
+                    }
+                }
+                auto pe   = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, nt + img.size() / (2 * config.head_dim));
+                auto pp   = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.pixel_attn_dim / config.pixel_heads / 2, (h / config.patch_size) * (w / config.patch_size));
+                auto pos  = ggml_new_tensor_3d(compute_ctx, GGML_TYPE_F32, config.pixel_dim, h * w, 1);
+                auto mask = ggml_new_tensor_2d(compute_ctx, GGML_TYPE_F32, nt, nt);
+                set_backend_tensor_data(pe, joint_pe_data.data());
+                set_backend_tensor_data(pp, pixel_pe_data.data());
+                set_backend_tensor_data(pos, pixel_pos_data.data());
+                set_backend_tensor_data(mask, text_mask_data.data());
+                auto ctx = get_context();
+                auto out = model.forward(&ctx, x, t, text, mask, pe, pp, pos);
+                ggml_build_forward_expand(gf, out);
+                return gf;
+            };
+            return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), inputs.x->dim());
+        }
+    };
+}  // namespace Iris
+
+#endif  // __SD_MODEL_DIFFUSION_IRIS_HPP__
diff --git a/src/model/diffusion/krea2.hpp b/src/model/diffusion/krea2.hpp
index 84ecfe0b1c..e58df41081 100644
--- a/src/model/diffusion/krea2.hpp
+++ b/src/model/diffusion/krea2.hpp
@@ -689,23 +689,28 @@ namespace Krea2 {
         }
     };
 
-    __STATIC_INLINE__ std::vector<float> gen_krea2_pe(int h,
-                                                      int w,
-                                                      int patch_size,
-                                                      int bs,
-                                                      int context_len,
-                                                      float theta,
-                                                      const std::vector<int>& axes_dim,
-                                                      const std::vector<ggml_tensor*>& ref_latents,
-                                                      Rope::RefIndexMode ref_index_mode) {
+    __STATIC_INLINE__ Rope::Embedding gen_krea2_pe(int h,
+                                                   int w,
+                                                   int patch_size,
+                                                   int bs,
+                                                   int context_len,
+                                                   float theta,
+                                                   const std::vector<int>& axes_dim,
+                                                   const std::vector<ggml_tensor*>& ref_latents,
+                                                   Rope::RefIndexMode ref_index_mode) {
+        Rope::Embedding result;
+        result.batch_size = bs;
+        result.positions.append_tokens(context_len);
         auto txt_ids = Rope::gen_flux_txt_ids(bs, context_len, 3, {});
-        auto img_ids = Rope::gen_flux_img_ids(h, w, patch_size, bs, 3, 0, 0, 0, false);
+        auto img_ids = Rope::gen_flux_img_ids(h, w, patch_size, bs, 3, 0, 0, 0, false, &result.positions);
         auto ids     = Rope::concat_ids(txt_ids, img_ids, bs);
         if (ref_latents.size() > 0) {
-            auto refs_ids = Rope::gen_refs_ids(patch_size, bs, 3, 1, ref_latents, ref_index_mode, 1.0f, false, 0);
+            auto refs_ids = Rope::gen_refs_ids(patch_size, bs, 3, 1, ref_latents, ref_index_mode, 1.0f, false, 0, &result.positions);
             ids           = Rope::concat_ids(ids, refs_ids, bs);
         }
-        return Rope::embed_nd(ids, bs, theta, axes_dim);
+        result.ids    = std::move(ids);
+        result.values = Rope::embed_nd(result.ids, bs, theta, axes_dim, result.layout, &result.frequencies);
+        return result;
     }
 
     struct Krea2Runner : public DiffusionModelRunner {
@@ -749,15 +754,15 @@ namespace Krea2 {
                 ref_latents.push_back(make_input(ref_latent_tensor));
             }
 
-            pe_vec      = gen_krea2_pe(static_cast<int>(x->ne[1]),
-                                       static_cast<int>(x->ne[0]),
-                                       config.patch_size,
-                                       static_cast<int>(x->ne[3]),
-                                       static_cast<int>(context->ne[1]),
-                                       config.theta,
-                                       config.axes_dim,
-                                       ref_latents,
-                                       ref_image_params.ref_index_mode);
+            pe_vec      = finish_rope_pe(gen_krea2_pe(static_cast<int>(x->ne[1]),
+                                                      static_cast<int>(x->ne[0]),
+                                                      config.patch_size,
+                                                      static_cast<int>(x->ne[3]),
+                                                      static_cast<int>(context->ne[1]),
+                                                      config.theta,
+                                                      config.axes_dim,
+                                                      ref_latents,
+                                                      ref_image_params.ref_index_mode));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
diff --git a/src/model/diffusion/lens.hpp b/src/model/diffusion/lens.hpp
index 35b4b517d4..8fb1512b41 100644
--- a/src/model/diffusion/lens.hpp
+++ b/src/model/diffusion/lens.hpp
@@ -384,14 +384,12 @@ namespace Lens {
             GGML_ASSERT(!context_tensor.empty());
             ggml_tensor* context = make_input(context_tensor);
 
-            pe_vec      = Rope::gen_lens_pe(static_cast<int>(x->ne[1]),
-                                            static_cast<int>(x->ne[0]),
-                                            static_cast<int>(x->ne[3]),
-                                            static_cast<int>(context->ne[1]),
-                                            config.theta,
-                                            circular_y_enabled,
-                                            circular_x_enabled,
-                                            config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_lens_pe(static_cast<int>(x->ne[1]),
+                                                           static_cast<int>(x->ne[0]),
+                                                           static_cast<int>(x->ne[3]),
+                                                           static_cast<int>(context->ne[1]),
+                                                           config.theta,
+                                                           config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
diff --git a/src/model/diffusion/llada_image.hpp b/src/model/diffusion/llada_image.hpp
index d03383a559..e69c3ecd39 100644
--- a/src/model/diffusion/llada_image.hpp
+++ b/src/model/diffusion/llada_image.hpp
@@ -412,16 +412,14 @@ namespace LLaDAImage {
             GGML_ASSERT(!context_tensor.empty());
             ggml_tensor* context = make_input(context_tensor);
 
-            pe_vec      = Rope::gen_llada_image_pe(static_cast<int>(x->ne[1]),
-                                                   static_cast<int>(x->ne[0]),
-                                                   config.patch_size,
-                                                   static_cast<int>(x->ne[3]),
-                                                   static_cast<int>(context->ne[1]),
-                                                   ZImage::SEQ_MULTI_OF,
-                                                   config.theta,
-                                                   circular_y_enabled,
-                                                   circular_x_enabled,
-                                                   config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_llada_image_pe(static_cast<int>(x->ne[1]),
+                                                                  static_cast<int>(x->ne[0]),
+                                                                  config.patch_size,
+                                                                  static_cast<int>(x->ne[3]),
+                                                                  static_cast<int>(context->ne[1]),
+                                                                  ZImage::SEQ_MULTI_OF,
+                                                                  config.theta,
+                                                                  config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
@@ -461,14 +459,14 @@ namespace LLaDAImage {
             ggml_tensor* source    = make_input(source_tensor);
             GGML_ASSERT(x->ne[3] == 1);
 
-            pe_vec      = Rope::gen_llada_image_edit_pe(static_cast<int>(x->ne[1]),
-                                                        static_cast<int>(x->ne[0]),
-                                                        config.patch_size,
-                                                        static_cast<int>(context->ne[1]),
-                                                   semantic != nullptr ? static_cast<int>(semantic->ne[1]) : 0,
-                                                        ZImage::SEQ_MULTI_OF,
-                                                        config.theta,
-                                                        config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_llada_image_edit_pe(static_cast<int>(x->ne[1]),
+                                                                       static_cast<int>(x->ne[0]),
+                                                                       config.patch_size,
+                                                                       static_cast<int>(context->ne[1]),
+                                                                  semantic != nullptr ? static_cast<int>(semantic->ne[1]) : 0,
+                                                                       ZImage::SEQ_MULTI_OF,
+                                                                       config.theta,
+                                                                       config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
diff --git a/src/model/diffusion/ltxv.hpp b/src/model/diffusion/ltxv.hpp
index 31a65afa65..e10d5e829a 100644
--- a/src/model/diffusion/ltxv.hpp
+++ b/src/model/diffusion/ltxv.hpp
@@ -548,21 +548,22 @@ namespace LTXV {
         return build_rope_matrix_from_frequencies(freqs, dim);
     }
 
-    __STATIC_INLINE__ ggml_tensor* apply_hidden_rope(ggml_context* ctx,
+    __STATIC_INLINE__ ggml_tensor* apply_hidden_rope(GGMLRunnerContext* runner_ctx,
                                                      ggml_tensor* x,
                                                      ggml_tensor* pe,
                                                      int64_t heads,
                                                      int64_t dim_head,
                                                      bool rope_interleaved) {
+        auto ctx = runner_ctx->ggml_ctx;
         GGML_ASSERT(x->ne[0] == heads * dim_head);
         auto x4 = ggml_reshape_4d(ctx, x, dim_head, heads, x->ne[1], x->ne[2]);
         if (pe != nullptr && pe->ne[3] == x->ne[1] * heads) {
             auto x_flat   = ggml_reshape_4d(ctx, x4, dim_head, 1, x->ne[1] * heads, x->ne[2]);
-            auto out_flat = Rope::apply_rope(ctx, x_flat, pe, rope_interleaved);
+            auto out_flat = Rope::apply_rope(runner_ctx, x_flat, pe, rope_interleaved);
             auto out4     = ggml_reshape_4d(ctx, out_flat, dim_head, heads, x->ne[1], x->ne[2]);
             return ggml_reshape_3d(ctx, out4, heads * dim_head, x->ne[1], x->ne[2]);
         }
-        return Rope::apply_rope(ctx, x4, pe, rope_interleaved);
+        return Rope::apply_rope(runner_ctx, x4, pe, rope_interleaved);
     }
 
     struct TimestepEmbedder : public GGMLBlock {
@@ -705,8 +706,8 @@ namespace LTXV {
                 if (k_pe == nullptr) {
                     k_pe = pe;
                 }
-                q = apply_hidden_rope(ctx->ggml_ctx, q, pe, heads, dim_head, rope_interleaved);
-                k = apply_hidden_rope(ctx->ggml_ctx, k, k_pe, heads, dim_head, rope_interleaved);
+                q = apply_hidden_rope(ctx, q, pe, heads, dim_head, rope_interleaved);
+                k = apply_hidden_rope(ctx, k, k_pe, heads, dim_head, rope_interleaved);
             }
 
             auto out = ggml_ext_attention_ext(ctx,
diff --git a/src/model/diffusion/mage_flow.hpp b/src/model/diffusion/mage_flow.hpp
index 89df0f4827..1b9b9f9be9 100644
--- a/src/model/diffusion/mage_flow.hpp
+++ b/src/model/diffusion/mage_flow.hpp
@@ -110,13 +110,13 @@ namespace MageFlow {
             }
 
             int batch_size = static_cast<int>(x->ne[3]);
-            pe_vec         = Rope::gen_mage_flow_pe(static_cast<int>(x->ne[1]),
-                                                    static_cast<int>(x->ne[0]),
-                                                    batch_size,
-                                                    static_cast<int>(context->ne[1]),
-                                                    ref_latents,
-                                                    config.theta,
-                                                    config.axes_dim);
+            pe_vec         = finish_rope_pe(Rope::gen_mage_flow_pe(static_cast<int>(x->ne[1]),
+                                                                   static_cast<int>(x->ne[0]),
+                                                                   batch_size,
+                                                                   static_cast<int>(context->ne[1]),
+                                                                   ref_latents,
+                                                                   config.theta,
+                                                                   config.axes_dim));
             int pos_len    = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             auto pe        = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
             set_backend_tensor_data(pe, pe_vec.data());
diff --git a/src/model/diffusion/ming_image.hpp b/src/model/diffusion/ming_image.hpp
new file mode 100644
index 0000000000..f19be99033
--- /dev/null
+++ b/src/model/diffusion/ming_image.hpp
@@ -0,0 +1,135 @@
+#ifndef __SD_MODEL_DIFFUSION_MING_IMAGE_HPP__
+#define __SD_MODEL_DIFFUSION_MING_IMAGE_HPP__
+
+#include "z_image.hpp"
+
+namespace MingImage {
+    struct MingImageConfig : ZImage::ZImageConfig {
+        bool split_qkv = true;
+
+        static MingImageConfig detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) {
+            MingImageConfig config;
+            static_cast<ZImage::ZImageConfig&>(config) = ZImage::ZImageConfig::detect_from_weights(tensors, prefix);
+            config.split_qkv                           = tensors.count(prefix + ".layers.0.attention.qkv.weight") == 0;
+            return config;
+        }
+    };
+
+    class MingImageModel : public GGMLBlock {
+        MingImageConfig config;
+
+    public:
+        explicit MingImageModel(const MingImageConfig& config)
+            : config(config) {
+            blocks["x_embedder"]     = std::make_shared<Linear>(config.patch_size * config.patch_size * config.in_channels, config.hidden_size);
+            blocks["t_embedder"]     = std::make_shared<TimestepEmbedder>(1024, 256, std::min<int64_t>(config.hidden_size, 256));
+            blocks["cap_embedder.0"] = std::make_shared<RMSNorm>(config.cap_feat_dim, config.norm_eps);
+            blocks["cap_embedder.1"] = std::make_shared<Linear>(config.cap_feat_dim, config.hidden_size);
+            auto add_blocks          = [&](const std::string& prefix, int64_t count, bool modulation) {
+                for (int64_t i = 0; i < count; ++i) {
+                    blocks[prefix + std::to_string(i)] = std::make_shared<ZImage::JointTransformerBlock>(
+                        static_cast<int>(i), config.hidden_size, config.head_dim, config.num_heads,
+                        config.num_kv_heads, config.multiple_of, config.ffn_dim_multiplier,
+                        config.norm_eps, config.qk_norm, modulation, true, config.split_qkv, 1e-5f);
+                }
+            };
+            add_blocks("noise_refiner.", config.num_refiner_layers, true);
+            add_blocks("context_refiner.", config.num_refiner_layers, false);
+            add_blocks("layers.", config.num_layers, true);
+            blocks["final_layer"] = std::make_shared<ZImage::FinalLayer>(config.hidden_size, config.patch_size, config.out_channels);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* timestep, ggml_tensor* context, ggml_tensor* direct, ggml_tensor* pe) {
+            auto gctx           = ctx->ggml_ctx;
+            const int64_t width = x->ne[0], height = x->ne[1];
+            auto img            = DiT::pad_and_patchify(ctx, x, config.patch_size, config.patch_size, false);
+            img                 = std::dynamic_pointer_cast<Linear>(blocks["x_embedder"])->forward(ctx, img);
+            auto txt            = std::dynamic_pointer_cast<RMSNorm>(blocks["cap_embedder.0"])->forward(ctx, context);
+            txt                 = std::dynamic_pointer_cast<Linear>(blocks["cap_embedder.1"])->forward(ctx, txt);
+            txt                 = ggml_concat(gctx, txt, direct, 1);
+            auto t              = std::dynamic_pointer_cast<TimestepEmbedder>(blocks["t_embedder"])->forward(ctx, timestep);
+            const int64_t n_txt = txt->ne[1], n_img = img->ne[1];
+            auto txt_pe = ggml_ext_slice(gctx, pe, 3, 0, n_txt);
+            auto img_pe = ggml_ext_slice(gctx, pe, 3, n_txt, n_txt + n_img);
+            for (int64_t i = 0; i < config.num_refiner_layers; ++i) {
+                txt = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["context_refiner." + std::to_string(i)])->forward(ctx, txt, txt_pe);
+                sd::ggml_graph_cut::mark_graph_cut(txt, "ming_image.context_refiner." + std::to_string(i), "txt");
+            }
+            for (int64_t i = 0; i < config.num_refiner_layers; ++i) {
+                img = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["noise_refiner." + std::to_string(i)])->forward(ctx, img, img_pe, nullptr, t);
+                sd::ggml_graph_cut::mark_graph_cut(img, "ming_image.noise_refiner." + std::to_string(i), "img");
+            }
+            auto combined = ggml_concat(gctx, txt, img, 1);
+            for (int64_t i = 0; i < config.num_layers; ++i) {
+                combined = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["layers." + std::to_string(i)])->forward(ctx, combined, pe, nullptr, t);
+                sd::ggml_graph_cut::mark_graph_cut(combined, "ming_image.layers." + std::to_string(i), "combined");
+            }
+            img = ggml_ext_slice(gctx, combined, 1, n_txt, n_txt + n_img);
+            img = std::dynamic_pointer_cast<ZImage::FinalLayer>(blocks["final_layer"])->forward(ctx, img, t);
+            img = DiT::unpatchify_and_crop(gctx, img, height, width, config.patch_size, config.patch_size, false);
+            return ggml_scale(gctx, img, -1.f);
+        }
+    };
+
+    struct MingImageRunner : DiffusionModelRunner {
+        MingImageConfig config;
+        MingImageModel model;
+        std::vector<float> pe_values;
+
+        MingImageRunner(ggml_backend_t backend, const String2TensorStorage& tensors, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : DiffusionModelRunner(backend, prefix, weight_manager),
+              config(MingImageConfig::detect_from_weights(tensors, prefix)),
+              model(config) {
+            model.init(params_ctx, tensors, prefix);
+        }
+
+        std::string get_desc() override { return "ming_image"; }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        sd::Tensor<float> compute(int n_threads, const DiffusionParams& inputs) override {
+            const auto* extra = diffusion_extra_as<MingImageDiffusionExtra>(inputs);
+            if (inputs.ref_latents != nullptr && !inputs.ref_latents->empty()) {
+                LOG_ERROR("Ming-Image reference-image conditioning is not supported");
+                return {};
+            }
+            if (inputs.context == nullptr || extra->direct_context == nullptr) {
+                LOG_ERROR("Ming-Image requires both query and direct text conditions");
+                return {};
+            }
+            auto graph = [&]() {
+                auto gf      = new_graph_custom(ZImage::Z_IMAGE_GRAPH_SIZE);
+                auto x       = make_input(*inputs.x);
+                auto t       = make_input(*inputs.timesteps);
+                auto context = make_input(*inputs.context);
+                auto direct  = make_input(*extra->direct_context);
+                GGML_ASSERT(x->ne[3] == 1);
+                const int64_t n_txt = context->ne[1] + direct->ne[1];
+                const int64_t n_img = ((x->ne[0] + config.patch_size - 1) / config.patch_size) *
+                                      ((x->ne[1] + config.patch_size - 1) / config.patch_size);
+                auto padded = finish_rope_pe(Rope::gen_z_image_pe(
+                    static_cast<int>(x->ne[1]), static_cast<int>(x->ne[0]), config.patch_size, 1,
+                    static_cast<int>(n_txt), ZImage::SEQ_MULTI_OF, {}, Rope::RefIndexMode::FIXED,
+                    config.theta, config.axes_dim));
+                // Zero-masked alignment tokens cannot affect valid queries. Omit them while
+                // retaining the padded caption length used to position image tokens.
+                const size_t stride      = config.axes_dim_sum * 2;
+                const int64_t padded_txt = n_txt + Rope::bound_mod(static_cast<int>(n_txt), ZImage::SEQ_MULTI_OF);
+                pe_values.assign(padded.begin(), padded.begin() + n_txt * stride);
+                pe_values.insert(pe_values.end(), padded.begin() + padded_txt * stride,
+                                 padded.begin() + (padded_txt + n_img) * stride);
+                auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, n_txt + n_img);
+                set_backend_tensor_data(pe, pe_values.data());
+                auto ctx = get_context();
+                auto out = model.forward(&ctx, x, t, context, direct, pe);
+                ggml_build_forward_expand(gf, out);
+                return gf;
+            };
+            return restore_trailing_singleton_dims(GGMLRunner::compute(graph, n_threads, false), inputs.x->dim());
+        }
+    };
+}
+
+#endif  // __SD_MODEL_DIFFUSION_MING_IMAGE_HPP__
diff --git a/src/model/diffusion/minimax_h3.hpp b/src/model/diffusion/minimax_h3.hpp
index 80c29ab528..7caaeef7c2 100644
--- a/src/model/diffusion/minimax_h3.hpp
+++ b/src/model/diffusion/minimax_h3.hpp
@@ -4,6 +4,7 @@
 #include <algorithm>
 #include <cinttypes>
 #include <cmath>
+#include <cstdlib>
 #include <set>
 #include <string>
 #include <tuple>
@@ -120,6 +121,54 @@ namespace MiniMaxH3 {
         }
     };
 
+    // Value-preserving graph rewrites for the DiT blocks. Every lever produces the same bits as
+    // the original graph (same arithmetic, same rounding points); each can be switched off:
+    //   SD_H3_GRAPH_FAST=0  restores the original graph for all of them
+    //   SD_H3_FAST_QKV=0    q/k/v: unfused chunk copies, partial RoPE via slice/rope/concat, K/V scale+cast
+    //   SD_H3_FAST_MLP=0    MLP gate: copied chunks + silu + mul instead of one swiglu on views
+    //   SD_H3_FAST_VIEWS=0  modulation/residual segments copied before use instead of read in place
+    //   SD_H3_FAST_SEGMENTS=0  per-segment adaLN modulation / gated residual as mul+add+concat chains
+    //                          instead of one in-place op per segment, MLP Linear scales kept as
+    //                          separate scale ops instead of folded into those ops and the swiglu
+    static bool env_flag(const char* name, bool default_value) {
+        const char* value = std::getenv(name);
+        if (value == nullptr || value[0] == '\0') {
+            return default_value;
+        }
+        return std::atoi(value) != 0;
+    }
+
+    static bool graph_fast() {
+        static const bool enabled = env_flag("SD_H3_GRAPH_FAST", true);
+        return enabled;
+    }
+
+    static bool fast_qkv() {
+        static const bool enabled = graph_fast() && env_flag("SD_H3_FAST_QKV", true);
+        return enabled;
+    }
+
+    static bool fast_mlp() {
+        static const bool enabled = graph_fast() && env_flag("SD_H3_FAST_MLP", true);
+        return enabled;
+    }
+
+    // SD_H3_FAST_SAGE_QKV=0: --sage-attn keeps the unfused q/k/v chain.
+    static bool fast_sage_qkv() {
+        static const bool enabled = fast_qkv() && env_flag("SD_H3_FAST_SAGE_QKV", true);
+        return enabled;
+    }
+
+    static bool fast_segments() {
+        static const bool enabled = graph_fast() && env_flag("SD_H3_FAST_SEGMENTS", true);
+        return enabled;
+    }
+
+    static bool fast_views() {
+        static const bool enabled = graph_fast() && env_flag("SD_H3_FAST_VIEWS", true);
+        return enabled;
+    }
+
     static float time_shift_sigma(float sigma, float from_shift, float to_shift) {
         float base = sigma / (from_shift + sigma * (1.f - from_shift));
         return to_shift * base / (1.f + (to_shift - 1.f) * base);
@@ -145,10 +194,57 @@ namespace MiniMaxH3 {
             blocks["fc2"] = std::make_shared<Linear>(ffn_hidden_size, hidden_size, false, false, true, 1.f / 128.f);
         }
 
+        // Scales for forward_folded(): the caller applies `input_scale` to the input (fused into the
+        // preceding op) and `output_scale` to the result (fused into the following op). Returns false
+        // when the Linears cannot be folded (bias, adapter, weight scales, INT8/FP8 weights).
+        bool foldable(GGMLRunnerContext* ctx, float* input_scale, float* output_scale) {
+            auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
+            auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
+            if (fc1->foldable_weight(ctx) == nullptr || fc2->foldable_weight(ctx) == nullptr) {
+                return false;
+            }
+            const float s1 = fc1->effective_scale(ctx);
+            const float s2 = fc2->effective_scale(ctx);
+            // the same factors ggml_ext_linear applies around each matmul
+            *input_scale  = s1;
+            *output_scale = s2 != 1.f ? 1.f / s2 : 1.f;
+            return true;
+        }
+
+        // fc1 matmul -> swiglu with fc1's output scale and fc2's input scale fused in -> fc2 matmul.
+        // x must already carry fc1's input scale; the result still needs foldable()'s output_scale.
+        ggml_tensor* forward_folded(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto fc1      = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
+            auto fc2      = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
+            const float s1 = fc1->effective_scale(ctx);
+            const float s2 = fc2->effective_scale(ctx);
+            auto h        = ggml_ext_linear_matmul(ctx->ggml_ctx, x, fc1->foldable_weight(ctx), fc1->prec_f32());
+            auto halves   = ggml_ext_chunk(ctx->ggml_ctx, h, 2, 0, false);
+#ifndef SD_GGML_H3_FUSED_OPS
+            GGML_UNUSED(s1);
+            GGML_UNUSED(s2);
+            GGML_UNUSED(halves);
+            GGML_ABORT("forward_folded needs the fused swiglu op");
+#else
+            auto gated = ggml_swiglu_scaled(ctx->ggml_ctx, halves[0], halves[1], s1 != 1.f ? 1.f / s1 : 1.f, s2);
+            return ggml_ext_linear_matmul(ctx->ggml_ctx, gated, fc2->foldable_weight(ctx), fc2->prec_f32());
+#endif
+        }
+
         ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
             auto fc1 = std::dynamic_pointer_cast<Linear>(blocks["fc1"]);
             auto fc2 = std::dynamic_pointer_cast<Linear>(blocks["fc2"]);
-            auto uv  = ggml_ext_chunk(ctx->ggml_ctx, fc1->forward(ctx, x), 2, 0);
+            auto h   = fc1->forward(ctx, x);
+            if (fast_mlp() && ctx->backend != nullptr && h->type == GGML_TYPE_F32 && ggml_is_contiguous(h)) {
+                // silu(u) * v straight from the two halves of fc1's output: same per-element
+                // silu and product as the copied-chunk graph, without the two chunk copies.
+                auto halves = ggml_ext_chunk(ctx->ggml_ctx, h, 2, 0, false);
+                auto gated  = ggml_swiglu_split(ctx->ggml_ctx, halves[0], halves[1]);
+                if (ggml_backend_supports_op(ctx->backend, gated)) {
+                    return fc2->forward(ctx, gated);
+                }
+            }
+            auto uv = ggml_ext_chunk(ctx->ggml_ctx, h, 2, 0);
             return fc2->forward(ctx, ggml_mul(ctx->ggml_ctx,
                                               ggml_silu(ctx->ggml_ctx, uv[0]),
                                               uv[1]));
@@ -160,12 +256,13 @@ namespace MiniMaxH3 {
         return ggml_reshape_3d(ctx, x, x->ne[0], x->ne[1], x->ne[2] * x->ne[3]);
     }
 
-    static ggml_tensor* apply_partial_rope(ggml_context* ctx,
+    static ggml_tensor* apply_partial_rope(GGMLRunnerContext* runner_ctx,
                                            ggml_tensor* x,
                                            ggml_tensor* pe) {
+        auto ctx        = runner_ctx->ggml_ctx;
         int64_t rot_dim = pe->ne[2] * 2;
         GGML_ASSERT(rot_dim <= x->ne[0]);
-        auto rotated = Rope::apply_rope(ctx,
+        auto rotated = Rope::apply_rope(runner_ctx,
                                         ggml_ext_slice(ctx, x, 0, 0, rot_dim),
                                         pe,
                                         false);
@@ -192,6 +289,80 @@ namespace MiniMaxH3 {
             blocks["out_proj"] = std::make_shared<Linear>(inner, hidden_size, false);
         }
 
+        // q/k/v read in place from the fused projection, q/k RMS-normed, then one op per tensor
+        // applies the partial RoPE, writes the head-major layout flash attention reads, and for
+        // K/V applies the kv scale and the F16 cast. Returns nullptr when unsupported.
+        ggml_tensor* forward_head_major(GGMLRunnerContext* ctx,
+                                        ggml_tensor* qkv_out,
+                                        int64_t sequence,
+                                        int64_t batch,
+                                        ggml_tensor* pe) {
+#ifndef SD_GGML_H3_FUSED_OPS
+            return nullptr;
+#else
+            const bool sage = ctx->sage_attn_enabled;
+            if (ctx->backend == nullptr || (sage ? !fast_sage_qkv() : !ctx->flash_attn_enabled)) {
+                return nullptr;
+            }
+            // the sage kernel takes 64 or 128 channel heads without padding
+            if (sage && head_dim != 64 && head_dim != 128) {
+                return nullptr;
+            }
+            const int64_t inner = heads * head_dim;
+            if (qkv_out->type != GGML_TYPE_F32 || !ggml_is_contiguous(qkv_out) || qkv_out->ne[0] != inner * 3 ||
+                qkv_out->ne[1] != sequence) {
+                return nullptr;
+            }
+            const int n_rot = pe != nullptr ? static_cast<int>(pe->ne[2] * 2) : 0;
+            if (n_rot > head_dim || (pe != nullptr && (pe->ne[0] != 2 || pe->ne[1] != 2 || pe->ne[3] != sequence ||
+                                                        pe->type != GGML_TYPE_F32 || !ggml_is_contiguous(pe)))) {
+                return nullptr;
+            }
+            const float kv_scale = ctx->attn_scale > 0.f ? ctx->attn_scale : 1.f / 128.f;
+
+            auto q_norm = std::dynamic_pointer_cast<RMSNorm>(blocks["q_norm"]);
+            auto k_norm = std::dynamic_pointer_cast<RMSNorm>(blocks["k_norm"]);
+            auto part   = [&](int index) {
+                return ggml_view_4d(ctx->ggml_ctx,
+                                    qkv_out,
+                                    head_dim,
+                                    heads,
+                                    sequence,
+                                    batch,
+                                    head_dim * ggml_element_size(qkv_out),
+                                    qkv_out->nb[1],
+                                    qkv_out->nb[1] * sequence,
+                                    index * inner * ggml_element_size(qkv_out));
+            };
+            auto q = q_norm->forward(ctx, part(0));
+            auto k = k_norm->forward(ctx, part(1));
+            // sage reads F32 Q/K and F16 V in this layout; K and V are scaled in F32, as on the unfused path
+            q      = ggml_rope_pe_permute(ctx->ggml_ctx, q, pe, n_rot, 1.f, GGML_TYPE_F32);
+            k      = ggml_rope_pe_permute(ctx->ggml_ctx, k, pe, n_rot, kv_scale, sage ? GGML_TYPE_F32 : GGML_TYPE_F16);
+            auto v = ggml_rope_pe_permute(ctx->ggml_ctx, part(2), nullptr, 0, kv_scale, GGML_TYPE_F16);
+            if (!ggml_backend_supports_op(ctx->backend, q) || !ggml_backend_supports_op(ctx->backend, k) ||
+                !ggml_backend_supports_op(ctx->backend, v)) {
+                return nullptr;
+            }
+            if (sage) {
+                // same softmax scale expression as ggml_ext_attention_ext's sage branch
+                const float scale = 1.0f / sqrt((float)head_dim);
+                auto out          = ggml_sage_attn(ctx->ggml_ctx, q, k, v, scale / kv_scale, GGML_SAGE_ATTN_AUTO);
+                if (!ggml_backend_supports_op(ctx->backend, out)) {
+                    return nullptr;
+                }
+                if (kv_scale != 1.0f) {
+                    out = ggml_ext_scale(ctx->ggml_ctx, out, 1.0f / kv_scale);
+                }
+                return ggml_reshape_3d(ctx->ggml_ctx, out, head_dim * heads, sequence, batch);  // [N, L, C]
+            }
+            q = ggml_reshape_3d(ctx->ggml_ctx, q, head_dim, sequence, heads * batch);
+            k = ggml_reshape_3d(ctx->ggml_ctx, k, head_dim, sequence, heads * batch);
+            v = ggml_reshape_3d(ctx->ggml_ctx, v, head_dim, sequence, heads * batch);
+            return ggml_ext_attention_prepared(ctx->ggml_ctx, ctx->backend, q, k, v, heads, batch, kv_scale);
+#endif
+        }
+
         ggml_tensor* forward(GGMLRunnerContext* ctx,
                              ggml_tensor* x,
                              ggml_tensor* pe = nullptr) {
@@ -202,15 +373,22 @@ namespace MiniMaxH3 {
 
             int64_t sequence = x->ne[1];
             int64_t batch    = x->ne[2] * x->ne[3];
-            auto qkv         = ggml_ext_chunk(ctx->ggml_ctx, qkv_proj->forward(ctx, x), 3, 0);
+            auto qkv_out     = qkv_proj->forward(ctx, x);
+            if (fast_qkv()) {
+                auto out = forward_head_major(ctx, qkv_out, sequence, batch, pe);
+                if (out != nullptr) {
+                    return out_proj->forward(ctx, out);
+                }
+            }
+            auto qkv = ggml_ext_chunk(ctx->ggml_ctx, qkv_out, 3, 0);
             auto q           = ggml_reshape_4d(ctx->ggml_ctx, qkv[0], head_dim, heads, sequence, batch);
             auto k           = ggml_reshape_4d(ctx->ggml_ctx, qkv[1], head_dim, heads, sequence, batch);
             auto v           = ggml_reshape_4d(ctx->ggml_ctx, qkv[2], head_dim, heads, sequence, batch);
             q                = q_norm->forward(ctx, q);
             k                = k_norm->forward(ctx, k);
             if (pe != nullptr) {
-                q = apply_partial_rope(ctx->ggml_ctx, q, pe);
-                k = apply_partial_rope(ctx->ggml_ctx, k, pe);
+                q = apply_partial_rope(ctx, q, pe);
+                k = apply_partial_rope(ctx, k, pe);
             } else {
                 q = attention_layout(ctx->ggml_ctx, q);
                 k = attention_layout(ctx->ggml_ctx, k);
@@ -264,6 +442,9 @@ namespace MiniMaxH3 {
             for (int64_t i = 0; i < num_layers; ++i) {
                 auto block = std::dynamic_pointer_cast<TokenRefinerBlock>(blocks["blocks." + std::to_string(i)]);
                 x          = block->forward(ctx, x);
+                sd::ggml_graph_cut::mark_graph_cut(x,
+                                                   "minimax_h3.token_refiner.blocks." + std::to_string(i),
+                                                   "hidden_states");
             }
             return std::dynamic_pointer_cast<RMSNorm>(blocks["final_norm"])->forward(ctx, x);
         }
@@ -331,8 +512,8 @@ namespace MiniMaxH3 {
                                                 projection,
                                                 hidden_size * expand,
                                                 timestep_rows * modalities);
-        auto selected         = ggml_ext_slice(ctx, reshaped, 1, row, row + 1);
-        return ggml_ext_chunk(ctx, selected, expand, 0);
+        auto selected         = ggml_ext_slice(ctx, reshaped, 1, row, row + 1, !fast_views());
+        return ggml_ext_chunk(ctx, selected, expand, 0, !fast_views());
     }
 
     static ggml_tensor* modulate_segments(ggml_context* ctx,
@@ -352,7 +533,7 @@ namespace MiniMaxH3 {
                                        expand,
                                        modalities,
                                        segment.modulation_row);
-            auto part = ggml_ext_slice(ctx, x, 1, segment.start, segment.end);
+            auto part = ggml_ext_slice(ctx, x, 1, segment.start, segment.end, !fast_views());
             part      = ggml_add(ctx,
                                  ggml_add(ctx, part, ggml_mul(ctx, part, mods[scale_index])),
                                  mods[shift_index]);
@@ -371,8 +552,8 @@ namespace MiniMaxH3 {
         ggml_tensor* out = nullptr;
         for (const auto& segment : segments) {
             auto mods = modulation_row(ctx, projection, hidden_size, 6, 3, segment.modulation_row);
-            auto base = ggml_ext_slice(ctx, x, 1, segment.start, segment.end);
-            auto add  = ggml_ext_slice(ctx, update, 1, segment.start, segment.end);
+            auto base = ggml_ext_slice(ctx, x, 1, segment.start, segment.end, !fast_views());
+            auto add  = ggml_ext_slice(ctx, update, 1, segment.start, segment.end, !fast_views());
             auto part = ggml_add(ctx, base, ggml_mul(ctx, add, mods[gate_index]));
             out       = out == nullptr ? part : ggml_concat(ctx, out, part, 1);
         }
@@ -400,11 +581,95 @@ namespace MiniMaxH3 {
                                                                             config.uses_adaln_curves());
         }
 
+        // The block with every per-segment modulation / gated residual as one in-place op per segment
+        // (no slice copies, no concat) and the MLP Linear scales folded into those ops and the
+        // swiglu. Same arithmetic and rounding as forward(); nullptr when unsupported.
+        ggml_tensor* forward_fused(GGMLRunnerContext* ctx,
+                                   ggml_tensor* x,
+                                   ggml_tensor* t_emb,
+                                   const std::vector<TokenModulationSpan>& segments,
+                                   ggml_tensor* pe) {
+#ifndef SD_GGML_H3_FUSED_OPS
+            return nullptr;
+#else
+            if (ctx->backend == nullptr || segments.empty() || x->type != GGML_TYPE_F32) {
+                return nullptr;
+            }
+            int64_t expected = 0;
+            for (const auto& segment : segments) {
+                if (segment.start != expected || segment.end <= segment.start) {
+                    return nullptr;
+                }
+                expected = segment.end;
+            }
+            if (expected != x->ne[1]) {
+                return nullptr;
+            }
+            auto norm1 = std::dynamic_pointer_cast<RMSNorm>(blocks["norm1"]);
+            auto norm2 = std::dynamic_pointer_cast<RMSNorm>(blocks["norm2"]);
+            auto attn  = std::dynamic_pointer_cast<Attention>(blocks["attn"]);
+            auto mlp   = std::dynamic_pointer_cast<MLP>(blocks["mlp"]);
+            auto adaln = std::dynamic_pointer_cast<AdaLayerNormModulation>(blocks["adaln_proj"]);
+            auto mods  = adaln->forward(ctx, t_emb);
+
+            auto vec = [&](const TokenModulationSpan& segment, int index) {
+                return modulation_row(ctx->ggml_ctx, mods, config.hidden_size, 6, 3, segment.modulation_row)[index];
+            };
+            {
+                auto probe_vec = vec(segments[0], 0);
+                auto probe     = ggml_modulate_rows(ctx->ggml_ctx, nullptr, x, probe_vec, probe_vec, 0, 1, 1.f);
+                auto probe_g   = ggml_gated_add_rows(ctx->ggml_ctx, nullptr, x, x, probe_vec, 0, 1, 1.f);
+                if (!ggml_backend_supports_op(ctx->backend, probe) || !ggml_backend_supports_op(ctx->backend, probe_g)) {
+                    return nullptr;
+                }
+            }
+            auto modulate = [&](ggml_tensor* value, int shift_index, int scale_index, float out_scale) {
+                ggml_tensor* out = nullptr;
+                for (const auto& segment : segments) {
+                    out = ggml_modulate_rows(ctx->ggml_ctx, out, value, vec(segment, scale_index), vec(segment, shift_index),
+                                             segment.start, segment.end, out_scale);
+                }
+                return out;
+            };
+            auto gated = [&](ggml_tensor* base, ggml_tensor* update, int gate_index, float update_scale) {
+                ggml_tensor* out = nullptr;
+                for (const auto& segment : segments) {
+                    out = ggml_gated_add_rows(ctx->ggml_ctx, out, base, update, vec(segment, gate_index),
+                                              segment.start, segment.end, update_scale);
+                }
+                return out;
+            };
+
+            auto h = modulate(norm1->forward(ctx, x), 0, 1, 1.f);
+            x      = gated(x, attn->forward(ctx, h, pe), 2, 1.f);
+
+            float mlp_in_scale  = 1.f;
+            float mlp_out_scale = 1.f;
+            bool fold           = fast_mlp() && mlp->foldable(ctx, &mlp_in_scale, &mlp_out_scale);
+            if (fold) {
+                auto probe = ggml_swiglu_scaled(ctx->ggml_ctx, x, x, 1.f, 1.f);
+                fold       = ggml_backend_supports_op(ctx->backend, probe);
+            }
+            if (fold) {
+                h = modulate(norm2->forward(ctx, x), 3, 4, mlp_in_scale);
+                return gated(x, mlp->forward_folded(ctx, h), 5, mlp_out_scale);
+            }
+            h = modulate(norm2->forward(ctx, x), 3, 4, 1.f);
+            return gated(x, mlp->forward(ctx, h), 5, 1.f);
+#endif
+        }
+
         ggml_tensor* forward(GGMLRunnerContext* ctx,
                              ggml_tensor* x,
                              ggml_tensor* t_emb,
                              const std::vector<TokenModulationSpan>& segments,
                              ggml_tensor* pe) {
+            if (fast_segments()) {
+                auto out = forward_fused(ctx, x, t_emb, segments, pe);
+                if (out != nullptr) {
+                    return out;
+                }
+            }
             auto norm1 = std::dynamic_pointer_cast<RMSNorm>(blocks["norm1"]);
             auto norm2 = std::dynamic_pointer_cast<RMSNorm>(blocks["norm2"]);
             auto attn  = std::dynamic_pointer_cast<Attention>(blocks["attn"]);
@@ -527,7 +792,11 @@ namespace MiniMaxH3 {
             GGML_ASSERT(context->ne[0] == config.text_dim);
             auto condition_proj = std::dynamic_pointer_cast<Linear>(blocks["condition_proj"]);
             auto token_refiner  = std::dynamic_pointer_cast<TokenRefiner>(blocks["token_refiner"]);
-            return token_refiner->forward(ctx, condition_proj->forward(ctx, context));
+            auto projected      = condition_proj->forward(ctx, context);
+            sd::ggml_graph_cut::mark_graph_cut(projected,
+                                               "minimax_h3.condition_proj",
+                                               "hidden_states");
+            return token_refiner->forward(ctx, projected);
         }
 
         ggml_tensor* time_embedding(GGMLRunnerContext* ctx,
diff --git a/src/model/diffusion/minit2i.hpp b/src/model/diffusion/minit2i.hpp
index 06c4db5a5c..0ddacf02eb 100644
--- a/src/model/diffusion/minit2i.hpp
+++ b/src/model/diffusion/minit2i.hpp
@@ -154,18 +154,26 @@ namespace MiniT2I {
         return Rope::flatten(Rope::rope(Rope::linspace(0.f, static_cast<float>(length - 1), length), head_dim, 10000.f));
     }
 
-    inline std::vector<float> make_vision_rope(int side, int head_dim) {
+    inline Rope::Embedding make_vision_rope(int side, int head_dim) {
         GGML_ASSERT(head_dim % 4 == 0);
         int dim     = head_dim / 2;
         int quarter = dim / 2;
         int length  = side * side;
+        Rope::Embedding result;
+        result.positions.append_image(side, side);
         std::vector<float> out(static_cast<size_t>(length) * (head_dim / 2) * 4);
         std::vector<float> freqs(quarter);
         for (int i = 0; i < quarter; ++i) {
             freqs[i] = 1.0f / std::pow(10000.0f, static_cast<float>(2 * i) / static_cast<float>(dim));
         }
+        for (int axis : {1, 2}) {
+            for (float frequency : freqs) {
+                result.frequencies.push_back({static_cast<size_t>(axis), frequency});
+            }
+        }
         for (int y = 0; y < side; ++y) {
             for (int x = 0; x < side; ++x) {
+                result.ids.push_back({0.f, static_cast<float>(y), static_cast<float>(x)});
                 int pos     = y * side + x;
                 size_t base = static_cast<size_t>(pos) * (head_dim / 2) * 4;
                 for (int i = 0; i < quarter; ++i) {
@@ -182,7 +190,8 @@ namespace MiniT2I {
                 }
             }
         }
-        return out;
+        result.values = std::move(out);
+        return result;
     }
 
     struct SwiGLUMlp : public GGMLBlock {
@@ -475,6 +484,8 @@ namespace MiniT2I {
         int64_t cached_txt_len                      = -1;
         int64_t cached_hidden_size                  = -1;
         int64_t cached_head_dim                     = -1;
+        bool cached_circular_x                      = false;
+        bool cached_circular_y                      = false;
 
         MiniT2IRunner(ggml_backend_t backend,
                       const String2TensorStorage& tensor_storage_map      = {},
@@ -521,6 +532,8 @@ namespace MiniT2I {
                 cached_txt_len == txt_len &&
                 cached_hidden_size == config.hidden_size &&
                 cached_head_dim == config.head_dim &&
+                cached_circular_x == circular_x_enabled &&
+                cached_circular_y == circular_y_enabled &&
                 cached_pos_embed != nullptr &&
                 cached_txt_pe != nullptr &&
                 cached_joint_pe != nullptr) {
@@ -531,7 +544,7 @@ namespace MiniT2I {
 
             auto pos_embed_vec = make_2d_sincos_pos_embed(static_cast<int>(img_side), static_cast<int>(config.hidden_size));
             auto txt_pe_vec    = make_text_rope(static_cast<int>(txt_len), static_cast<int>(config.head_dim));
-            auto img_pe_vec    = make_vision_rope(static_cast<int>(img_side), static_cast<int>(config.head_dim));
+            auto img_pe_vec    = finish_rope_pe(make_vision_rope(static_cast<int>(img_side), static_cast<int>(config.head_dim)));
             auto joint_pe_vec  = txt_pe_vec;
             joint_pe_vec.insert(joint_pe_vec.end(), img_pe_vec.begin(), img_pe_vec.end());
 
@@ -561,6 +574,8 @@ namespace MiniT2I {
             cached_txt_len     = txt_len;
             cached_hidden_size = config.hidden_size;
             cached_head_dim    = config.head_dim;
+            cached_circular_x  = circular_x_enabled;
+            cached_circular_y  = circular_y_enabled;
         }
 
         ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
diff --git a/src/model/diffusion/model.hpp b/src/model/diffusion/model.hpp
index fb4e362c86..955c5277a7 100644
--- a/src/model/diffusion/model.hpp
+++ b/src/model/diffusion/model.hpp
@@ -7,7 +7,7 @@
 
 #include "core/ggml_runner.h"
 #include "core/tensor_ggml.hpp"
-#include "model/common/rope.hpp"
+#include "model/common/rope_circular.hpp"
 #include "model_manager.h"
 
 enum class RefImageResizeMode {
@@ -64,6 +64,10 @@ struct FluxDiffusionExtra {
     float pulid_id_weight               = 1.0f;
 };
 
+struct Flux3DiffusionExtra {
+    const sd::Tensor<float>* target_timestep = nullptr;
+};
+
 struct AnimaDiffusionExtra {
     const sd::Tensor<int32_t>* t5_ids   = nullptr;
     const sd::Tensor<float>* t5_weights = nullptr;
@@ -71,6 +75,8 @@ struct AnimaDiffusionExtra {
 
 struct QwenImage21DiffusionExtra {
     const sd::Tensor<int32_t>* image_slots = nullptr;
+    // Nonzero IDs identify immutable prefix inputs within one sampling run.
+    uint64_t prefix_id = 0;
 };
 
 struct WanDiffusionExtra {
@@ -139,10 +145,15 @@ struct LLaDAImageDiffusionExtra {
     const sd::Tensor<float>* semantic = nullptr;
 };
 
+struct MingImageDiffusionExtra {
+    const sd::Tensor<float>* direct_context = nullptr;
+};
+
 using DiffusionExtraParams = std::variant<std::monostate,
                                           UNetDiffusionExtra,
                                           SkipLayerDiffusionExtra,
                                           FluxDiffusionExtra,
+                                          Flux3DiffusionExtra,
                                           AnimaDiffusionExtra,
                                           QwenImage21DiffusionExtra,
                                           WanDiffusionExtra,
@@ -152,7 +163,8 @@ using DiffusionExtraParams = std::variant<std::monostate,
                                           MiniT2IDiffusionExtra,
                                           SenseNovaU1DiffusionExtra,
                                           HunyuanVideoDiffusionExtra,
-                                          LLaDAImageDiffusionExtra>;
+                                          LLaDAImageDiffusionExtra,
+                                          MingImageDiffusionExtra>;
 
 struct DiffusionParams {
     const sd::Tensor<float>* x                        = nullptr;
@@ -182,6 +194,11 @@ struct DiffusionModelRunner : public GGMLRunner {
 protected:
     std::string prefix;
 
+    std::vector<float> finish_rope_pe(Rope::Embedding embedding) {
+        Rope::apply_circular(embedding, circular_x_enabled, circular_y_enabled);
+        return std::move(embedding.values);
+    }
+
 public:
     DiffusionModelRunner(ggml_backend_t backend,
                          const std::string& prefix,
diff --git a/src/model/diffusion/moss_image.hpp b/src/model/diffusion/moss_image.hpp
new file mode 100644
index 0000000000..f1cbbc7f37
--- /dev/null
+++ b/src/model/diffusion/moss_image.hpp
@@ -0,0 +1,262 @@
+#ifndef __SD_MODEL_DIFFUSION_MOSS_IMAGE_HPP__
+#define __SD_MODEL_DIFFUSION_MOSS_IMAGE_HPP__
+
+#include <algorithm>
+#include <cinttypes>
+
+#include "model/common/ggml_block.hpp"
+#include "model/common/rope.hpp"
+#include "model/diffusion/dit.hpp"
+#include "model/diffusion/model.hpp"
+#include "model_loader.h"
+
+namespace MOSSImage {
+    struct MOSSImageConfig {
+        int64_t dim      = 3072;
+        int64_t ffn_dim  = 8192;
+        int64_t text_dim = 4096;
+        int64_t channels = 128;
+        int64_t head_dim = 128;
+        int64_t depth    = 42;
+        int64_t freq_dim = 256;
+
+        static MOSSImageConfig detect_from_weights(const String2TensorStorage& weights, const std::string& prefix) {
+            MOSSImageConfig config;
+            int64_t depth = 0;
+            for (const auto& [name, tensor] : weights) {
+                if (!starts_with(name, prefix + ".")) {
+                    continue;
+                }
+                auto key = name.substr(prefix.size() + 1);
+                if (key == "video_embedder.weight") {
+                    config.channels = tensor.ne[0];
+                    config.dim      = tensor.ne[1];
+                } else if (key == "text_embedder.proj.weight") {
+                    config.text_dim = tensor.ne[0];
+                } else if (key == "layers.0.proj.ffn_down.weight") {
+                    config.ffn_dim = tensor.ne[0];
+                } else if (key == "layers.0.q_norm.weight") {
+                    config.head_dim = tensor.ne[0];
+                } else if (key == "time_embedding.0.weight") {
+                    config.freq_dim = tensor.ne[0];
+                }
+                if (starts_with(key, "layers.")) {
+                    depth = std::max<int64_t>(depth, std::stoi(key.substr(7)) + 1);
+                }
+            }
+            if (depth) {
+                config.depth = depth;
+                LOG_VERBOSE("moss_image: dim = %" PRId64 ", ffn_dim = %" PRId64 ", depth = %" PRId64 ", head_dim = %" PRId64,
+                            config.dim, config.ffn_dim, config.depth, config.head_dim);
+            }
+            return config;
+        }
+    };
+
+    inline ggml_tensor* situ(ggml_context* c, ggml_tensor* hidden, ggml_tensor* gate) {
+        auto a = ggml_scale(c, ggml_tanh(c, ggml_scale(c, gate, 0.25f)), 4.f);
+        a      = ggml_mul(c, a, ggml_sigmoid(c, gate));
+        auto b = ggml_scale(c, ggml_tanh(c, ggml_scale(c, hidden, 1.f / 25.f)), 25.f);
+        return ggml_mul(c, a, b);
+    }
+
+    struct TextBlock : public GGMLBlock {
+        MOSSImageConfig config;
+
+        TextBlock(const MOSSImageConfig& config)
+            : config(config) {
+            blocks["norm1"]    = std::make_shared<RMSNorm>(config.dim, 1e-6f);
+            blocks["norm2"]    = std::make_shared<RMSNorm>(config.dim, 1e-6f);
+            blocks["q_norm"]   = std::make_shared<RMSNorm>(config.head_dim, 1e-6f);
+            blocks["k_norm"]   = std::make_shared<RMSNorm>(config.head_dim, 1e-6f);
+            blocks["qkv"]      = std::make_shared<Linear>(config.dim, config.dim * 3, true);
+            blocks["proj"]     = std::make_shared<Linear>(config.dim, config.dim, true);
+            blocks["ffn_up"]   = std::make_shared<Linear>(config.dim, config.ffn_dim, true);
+            blocks["ffn_gate"] = std::make_shared<Linear>(config.dim, config.ffn_dim, true);
+            blocks["ffn_down"] = std::make_shared<Linear>(config.ffn_dim, config.dim, true);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe) {
+            auto c      = ctx->ggml_ctx;
+            auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, h); };
+            auto norm   = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<RMSNorm>(blocks[name])->forward(ctx, h); };
+            auto y      = norm("norm1", x);
+            auto qkv    = linear("qkv", y);
+            auto split  = [&](int i) {
+                return ggml_reshape_4d(c, ggml_ext_slice(c, qkv, 0, i * config.dim, (i + 1) * config.dim), config.head_dim, config.dim / config.head_dim, x->ne[1], x->ne[2]);
+            };
+            auto q = norm("q_norm", split(0));
+            auto k = norm("k_norm", split(1));
+            auto v = split(2);
+            y      = Rope::attention(ctx, q, k, v, pe, nullptr, 1.f, false);
+            y      = linear("proj", y);
+            x      = ggml_add(c, x, y);
+            y      = norm("norm2", x);
+            y      = situ(c, linear("ffn_up", y), linear("ffn_gate", y));
+            y      = linear("ffn_down", y);
+            return ggml_add(c, x, y);
+        }
+    };
+
+    struct Block : public GGMLBlock {
+        MOSSImageConfig config;
+
+        Block(const MOSSImageConfig& config)
+            : config(config) {
+            blocks["norm1"]            = std::make_shared<RMSNorm>(config.dim, 1e-6f);
+            blocks["norm2"]            = std::make_shared<RMSNorm>(config.dim, 1e-6f);
+            blocks["q_norm"]           = std::make_shared<RMSNorm>(config.head_dim, 1e-6f);
+            blocks["k_norm"]           = std::make_shared<RMSNorm>(config.head_dim, 1e-6f);
+            blocks["proj.qkv"]         = std::make_shared<Linear>(config.dim, config.dim * 3, false);
+            blocks["proj.o"]           = std::make_shared<Linear>(config.dim, config.dim, false);
+            blocks["proj.ffn_up_gate"] = std::make_shared<Linear>(config.dim, config.ffn_dim * 2, false);
+            blocks["proj.ffn_down"]    = std::make_shared<Linear>(config.ffn_dim, config.dim, false);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, ggml_tensor* mod) {
+            auto c      = ctx->ggml_ctx;
+            auto linear = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, h); };
+            auto norm   = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<RMSNorm>(blocks[name])->forward(ctx, h); };
+            auto part   = [&](int i) { return ggml_ext_slice(c, mod, 0, i * config.dim, (i + 1) * config.dim); };
+            auto y      = norm("norm1", x);
+            y           = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(1))), part(0));
+            auto qkv    = linear("proj.qkv", y);
+            auto split  = [&](int i) {
+                return ggml_reshape_4d(c, ggml_ext_slice(c, qkv, 0, i * config.dim, (i + 1) * config.dim), config.head_dim, config.dim / config.head_dim, x->ne[1], x->ne[2]);
+            };
+            auto q       = norm("q_norm", split(0));
+            auto k       = norm("k_norm", split(1));
+            auto v       = split(2);
+            y            = Rope::attention(ctx, q, k, v, pe, nullptr, 1.f, false);
+            y            = linear("proj.o", y);
+            y            = ggml_mul(c, y, part(2));
+            x            = ggml_add(c, x, y);
+            y            = norm("norm2", x);
+            y            = ggml_add(c, ggml_add(c, y, ggml_mul(c, y, part(4))), part(3));
+            auto up_gate = linear("proj.ffn_up_gate", y);
+            y            = situ(c, ggml_ext_slice(c, up_gate, 0, 0, config.ffn_dim), ggml_ext_slice(c, up_gate, 0, config.ffn_dim, 2 * config.ffn_dim));
+            y            = linear("proj.ffn_down", y);
+            y            = ggml_mul(c, y, part(5));
+            return ggml_add(c, x, y);
+        }
+    };
+
+    struct Model : public GGMLBlock {
+        MOSSImageConfig config;
+
+        Model(const MOSSImageConfig& config)
+            : config(config) {
+            blocks["video_embedder"]     = std::make_shared<Linear>(config.channels, config.dim, true);
+            blocks["text_embedder.proj"] = std::make_shared<Linear>(config.text_dim, config.dim, true);
+            for (int i = 0; i < 2; ++i) {
+                blocks["text_embedder.blocks." + std::to_string(i)] = std::make_shared<TextBlock>(config);
+            }
+            blocks["time_embedding.0"]        = std::make_shared<Linear>(config.freq_dim, config.dim, true);
+            blocks["time_embedding.2"]        = std::make_shared<Linear>(config.dim, config.dim, true);
+            blocks["shared_block_modulation"] = std::make_shared<Linear>(config.dim, 6 * config.dim, true);
+            blocks["video_head_modulation"]   = std::make_shared<Linear>(config.dim, 2 * config.dim, true);
+            for (int64_t i = 0; i < config.depth; ++i) {
+                blocks["layers." + std::to_string(i)] = std::make_shared<Block>(config);
+            }
+            blocks["final_norm_video"]   = std::make_shared<RMSNorm>(config.dim, 1e-6f);
+            blocks["final_linear_video"] = std::make_shared<Linear>(config.dim, config.channels, false);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image, ggml_tensor* time, ggml_tensor* text, ggml_tensor* pe, const std::vector<int>& skip) {
+            auto c            = ctx->ggml_ctx;
+            auto linear       = [&](const char* name, ggml_tensor* h) { return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, h); };
+            int64_t image_len = image->ne[0] * image->ne[1];
+            auto x            = linear("video_embedder", DiT::patchify(c, image, 1, 1));
+            text              = linear("text_embedder.proj", text);
+            auto text_pe      = ggml_ext_slice(c, pe, 3, image_len, image_len + text->ne[1]);
+            for (int i = 0; i < 2; ++i) {
+                text = std::dynamic_pointer_cast<TextBlock>(blocks["text_embedder.blocks." + std::to_string(i)])->forward(ctx, text, text_pe);
+            }
+            time      = linear("time_embedding.2", ggml_silu(c, linear("time_embedding.0", time)));
+            auto mod  = linear("shared_block_modulation", time);
+            auto head = linear("video_head_modulation", time);
+            x         = ggml_concat(c, x, text, 1);
+            for (int64_t i = 0; i < config.depth; ++i) {
+                if (std::find(skip.begin(), skip.end(), i) == skip.end()) {
+                    x = std::dynamic_pointer_cast<Block>(blocks["layers." + std::to_string(i)])->forward(ctx, x, pe, mod);
+                    sd::ggml_graph_cut::mark_graph_cut(x, "moss_image.layers." + std::to_string(i), "x");
+                }
+            }
+            x          = ggml_ext_slice(c, x, 1, 0, image_len);
+            x          = std::dynamic_pointer_cast<RMSNorm>(blocks["final_norm_video"])->forward(ctx, x);
+            auto shift = ggml_ext_slice(c, head, 0, 0, config.dim);
+            auto scale = ggml_ext_slice(c, head, 0, config.dim, 2 * config.dim);
+            x          = ggml_add(c, ggml_add(c, x, ggml_mul(c, x, scale)), shift);
+            x          = linear("final_linear_video", x);
+            return DiT::unpatchify(c, x, image->ne[1], image->ne[0], 1, 1);
+        }
+    };
+
+    struct MOSSImageRunner : public DiffusionModelRunner {
+        MOSSImageConfig config;
+        Model model;
+        std::vector<float> pe_data, time_data;
+
+        MOSSImageRunner(ggml_backend_t backend, const String2TensorStorage& weights, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : DiffusionModelRunner(backend, prefix, weight_manager), config(MOSSImageConfig::detect_from_weights(weights, prefix)), model(config) {
+            model.init(params_ctx, weights, prefix);
+        }
+
+        std::string get_desc() override { return "MOSS-Image"; }
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        sd::Tensor<float> compute(int n_threads, const DiffusionParams& inputs) override {
+            if (!inputs.x || !inputs.context || !inputs.timesteps || inputs.context->empty()) {
+                LOG_ERROR("MOSS-Image requires latents, timesteps and text features");
+                return {};
+            }
+            std::vector<int> skip;
+            if (auto extra = std::get_if<SkipLayerDiffusionExtra>(&inputs.extra)) {
+                if (extra->skip_layers) {
+                    skip = *extra->skip_layers;
+                }
+            }
+            auto graph = [&]() {
+                auto gf   = new_graph_custom(65536);
+                auto x    = make_input(*inputs.x);
+                auto text = make_input(*inputs.context);
+                GGML_ASSERT(x->ne[3] == 1 && x->ne[2] == config.channels && text->ne[0] == config.text_dim);
+                int64_t h = x->ne[1], w = x->ne[0], nt = text->ne[1];
+                Rope::Embedding embedding;
+                embedding.positions.append_image(static_cast<int>(h), static_cast<int>(w));
+                embedding.positions.images.back().height_axis = 2;
+                embedding.positions.images.back().width_axis  = 3;
+                embedding.positions.append_tokens(nt);
+                for (int64_t token = 0; token < h * w + nt; ++token) {
+                    float pos[4] = {0.f, float(token - h * w), 0.f, 0.f};
+                    if (token < h * w) {
+                        pos[1] = float(nt);
+                        pos[2] = float(token / w) - float(h - 1) / 2.f;
+                        pos[3] = float(token % w) - float(w - 1) / 2.f;
+                    }
+                    embedding.ids.push_back({pos[0], pos[1], pos[2], pos[3]});
+                }
+                embedding.values = Rope::embed_nd(embedding.ids, 1, 10000.f, std::vector<int>(4, static_cast<int>(config.head_dim / 4)), embedding.layout, &embedding.frequencies);
+                pe_data          = finish_rope_pe(std::move(embedding));
+                time_data.resize(config.freq_dim);
+                for (int64_t i = 0; i < config.freq_dim / 2; ++i) {
+                    double angle                       = inputs.timesteps->values()[0] * std::pow(10000.0, -double(i) / double(config.freq_dim / 2));
+                    time_data[i]                       = float(std::cos(angle));
+                    time_data[i + config.freq_dim / 2] = float(std::sin(angle));
+                }
+                auto pe   = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, h * w + nt);
+                auto time = ggml_new_tensor_3d(compute_ctx, GGML_TYPE_F32, config.freq_dim, 1, 1);
+                set_backend_tensor_data(pe, pe_data.data());
+                set_backend_tensor_data(time, time_data.data());
+                auto ctx = get_context();
+                ggml_build_forward_expand(gf, model.forward(&ctx, x, time, text, pe, skip));
+                return gf;
+            };
+            return restore_trailing_singleton_dims(GGMLRunner::compute(graph, n_threads, false), inputs.x->dim());
+        }
+    };
+}  // namespace MOSSImage
+
+#endif  // __SD_MODEL_DIFFUSION_MOSS_IMAGE_HPP__
diff --git a/src/model/diffusion/pid.hpp b/src/model/diffusion/pid.hpp
index 59eac29503..6edc83422d 100644
--- a/src/model/diffusion/pid.hpp
+++ b/src/model/diffusion/pid.hpp
@@ -135,13 +135,13 @@ namespace Pid {
         return Rope::flatten(Rope::rope(Rope::linspace(0.f, static_cast<float>(length - 1), length), dim, theta));
     }
 
-    inline std::vector<float> make_rope_2d(int height,
-                                           int width,
-                                           int dim,
-                                           float theta    = 10000.f,
-                                           float scale    = 16.f,
-                                           int ref_grid_h = 0,
-                                           int ref_grid_w = 0) {
+    inline Rope::Embedding make_rope_2d(int height,
+                                        int width,
+                                        int dim,
+                                        float theta    = 10000.f,
+                                        float scale    = 16.f,
+                                        int ref_grid_h = 0,
+                                        int ref_grid_w = 0) {
         GGML_ASSERT(dim % 4 == 0);
         return Rope::embed_2d_interleaved(height, width, dim, theta, scale, ref_grid_h, ref_grid_w);
     }
@@ -867,13 +867,13 @@ namespace Pid {
             int64_t Hs = Hp / config.patch_size;
             int64_t Ws = Wp / config.patch_size;
 
-            pos_img_vec  = make_rope_2d(static_cast<int>(Hs),
-                                        static_cast<int>(Ws),
-                                        static_cast<int>(config.hidden_size / config.num_groups),
-                                        10000.f,
-                                        16.f,
-                                        static_cast<int>(config.rope_ref_grid_h),
-                                        static_cast<int>(config.rope_ref_grid_w));
+            pos_img_vec  = finish_rope_pe(make_rope_2d(static_cast<int>(Hs),
+                                                       static_cast<int>(Ws),
+                                                       static_cast<int>(config.hidden_size / config.num_groups),
+                                                       10000.f,
+                                                       16.f,
+                                                       static_cast<int>(config.rope_ref_grid_h),
+                                                       static_cast<int>(config.rope_ref_grid_w)));
             auto pos_img = ggml_new_tensor_4d(compute_ctx,
                                               GGML_TYPE_F32,
                                               2,
@@ -904,13 +904,13 @@ namespace Pid {
                                                 1);
             set_backend_tensor_data(pixel_pos, pixel_pos_vec.data());
 
-            pixel_pos_comp_vec  = make_rope_2d(static_cast<int>(Hs),
-                                               static_cast<int>(Ws),
-                                               static_cast<int>(config.pixel_attn_hidden_size / config.pixel_num_groups),
-                                               10000.f,
-                                               16.f,
-                                               static_cast<int>(config.rope_ref_grid_h),
-                                               static_cast<int>(config.rope_ref_grid_w));
+            pixel_pos_comp_vec  = finish_rope_pe(make_rope_2d(static_cast<int>(Hs),
+                                                              static_cast<int>(Ws),
+                                                              static_cast<int>(config.pixel_attn_hidden_size / config.pixel_num_groups),
+                                                              10000.f,
+                                                              16.f,
+                                                              static_cast<int>(config.rope_ref_grid_h),
+                                                              static_cast<int>(config.rope_ref_grid_w)));
             auto pixel_pos_comp = ggml_new_tensor_4d(compute_ctx,
                                                      GGML_TYPE_F32,
                                                      2,
diff --git a/src/model/diffusion/pixart.hpp b/src/model/diffusion/pixart.hpp
new file mode 100644
index 0000000000..cc06291928
--- /dev/null
+++ b/src/model/diffusion/pixart.hpp
@@ -0,0 +1,399 @@
+#ifndef __SD_MODEL_DIFFUSION_PIXART_HPP__
+#define __SD_MODEL_DIFFUSION_PIXART_HPP__
+
+#include <algorithm>
+#include <cinttypes>
+#include <cmath>
+#include <vector>
+
+#include "core/ggml_extend.h"
+#include "core/ggml_runner.h"
+#include "core/util.h"
+#include "model/common/ggml_block.hpp"
+#include "model/diffusion/dit.hpp"
+#include "model/diffusion/mmdit.hpp"
+#include "model/diffusion/model.hpp"
+#include "model_loader.h"
+
+// Ref: https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/pixart_transformer_2d.py
+// Ref: https://github.com/PixArt-alpha/PixArt-sigma
+
+namespace PixArt {
+    constexpr int PIXART_GRAPH_SIZE = 20480;
+    constexpr int ADALN_EMBED_DIM   = 256;
+
+    struct PixArtConfig {
+        int64_t in_channels         = 4;
+        int64_t out_channels        = 8;  // learn_sigma: noise prediction + learned variance
+        int64_t hidden_size         = 1152;
+        int64_t cross_attention_dim = 1152;
+        int64_t caption_channels    = 4096;
+        int64_t num_heads           = 16;
+        int64_t patch_size          = 2;
+        int64_t ffn_dim             = 4608;
+        int64_t pos_embed_base_size = 64;
+        float interpolation_scale   = 2.f;
+        int num_layers              = 28;
+
+        static PixArtConfig detect_from_weights(const String2TensorStorage& weights, const std::string& prefix) {
+            PixArtConfig config;
+            auto find = [&](const std::string& suffix) -> const TensorStorage* {
+                auto it = weights.find(prefix + "." + suffix);
+                return it == weights.end() ? nullptr : &it->second;
+            };
+            if (auto w = find("x_embedder.proj.weight")) {
+                config.hidden_size = w->ne[3];
+                config.in_channels = w->ne[2];
+                config.patch_size  = w->ne[0];
+            }
+            if (auto w = find("final_layer.linear.weight")) {
+                config.out_channels = w->ne[1] / (config.patch_size * config.patch_size);
+            }
+            if (auto w = find("y_embedder.y_proj.fc1.weight")) {
+                config.caption_channels = w->ne[0];
+            }
+            if (auto w = find("blocks.0.cross_attn.kv_linear.weight")) {
+                config.cross_attention_dim = w->ne[0];
+            }
+            if (auto w = find("blocks.0.mlp.fc1.weight")) {
+                config.ffn_dim = w->ne[1];
+            }
+            if (find("csize_embedder.mlp.0.weight") != nullptr) {
+                LOG_WARN("pixart: resolution/aspect-ratio micro conditions are not supported; output may differ from the reference");
+            }
+            int layers                     = 0;
+            const std::string block_prefix = prefix + ".blocks.";
+            for (const auto& [name, _] : weights) {
+                if (starts_with(name, block_prefix)) {
+                    layers = std::max(layers, atoi(name.substr(block_prefix.size()).c_str()) + 1);
+                }
+            }
+            if (layers > 0) {
+                config.num_layers = layers;
+                LOG_VERBOSE("pixart: layers = %d, hidden_size = %" PRId64,
+                            layers, config.hidden_size);
+            }
+            return config;
+        }
+    };
+
+    // Mirrors diffusers get_2d_sincos_pos_embed for a (gh, gw) patch grid.
+    static std::vector<float> gen_2d_sincos_pos_embed(int64_t dim,
+                                                      int64_t gh,
+                                                      int64_t gw,
+                                                      int64_t base_size,
+                                                      float interpolation_scale) {
+        // diffusers: meshgrid(grid_w, grid_h, indexing="xy") -> grid[0]=w, grid[1]=h,
+        // embedding = concat(sincos(w), sincos(h))
+        std::vector<float> out(static_cast<size_t>(gh) * gw * dim);
+        int64_t quarter = dim / 4;
+        for (int64_t h = 0; h < gh; ++h) {
+            float pos_h = static_cast<float>(h) / (static_cast<float>(gh) / base_size) / interpolation_scale;
+            for (int64_t w = 0; w < gw; ++w) {
+                float pos_w  = static_cast<float>(w) / (static_cast<float>(gw) / base_size) / interpolation_scale;
+                float* dst_w = out.data() + (h * gw + w) * dim;
+                float* dst_h = dst_w + dim / 2;
+                for (int64_t i = 0; i < quarter; ++i) {
+                    float omega        = 1.f / powf(10000.f, static_cast<float>(i) / quarter);
+                    dst_w[i]           = sinf(pos_w * omega);
+                    dst_w[i + quarter] = cosf(pos_w * omega);
+                    dst_h[i]           = sinf(pos_h * omega);
+                    dst_h[i + quarter] = cosf(pos_h * omega);
+                }
+            }
+        }
+        return out;
+    }
+
+    class PixArtTimestepEmbedding : public GGMLBlock {
+    public:
+        PixArtTimestepEmbedding(int64_t in_channels, int64_t out_dim) {
+            blocks["mlp.0"] = std::make_shared<Linear>(in_channels, out_dim);
+            blocks["mlp.2"] = std::make_shared<Linear>(out_dim, out_dim);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            x = std::dynamic_pointer_cast<Linear>(blocks["mlp.0"])->forward(ctx, x);
+            x = ggml_silu(ctx->ggml_ctx, x);
+            return std::dynamic_pointer_cast<Linear>(blocks["mlp.2"])->forward(ctx, x);
+        }
+    };
+
+    class PixArtAttention : public GGMLBlock {
+        int64_t num_heads;
+        bool self_attention;
+
+    public:
+        PixArtAttention(int64_t dim, int64_t num_heads, int64_t context_dim, bool self_attention)
+            : num_heads(num_heads), self_attention(self_attention) {
+            if (self_attention) {
+                blocks["qkv"] = std::make_shared<Linear>(dim, 3 * dim);
+            } else {
+                blocks["q_linear"]  = std::make_shared<Linear>(dim, dim);
+                blocks["kv_linear"] = std::make_shared<Linear>(context_dim, 2 * dim);
+            }
+            blocks["proj"] = std::make_shared<Linear>(dim, dim);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* context, ggml_tensor* mask = nullptr) {
+            std::vector<ggml_tensor*> qkv;
+            if (self_attention) {
+                auto projected = std::dynamic_pointer_cast<Linear>(blocks["qkv"])->forward(ctx, x);
+                qkv            = ggml_ext_chunk(ctx->ggml_ctx, projected, 3, 0);
+            } else {
+                auto q     = std::dynamic_pointer_cast<Linear>(blocks["q_linear"])->forward(ctx, x);
+                auto kv    = std::dynamic_pointer_cast<Linear>(blocks["kv_linear"])->forward(ctx, context);
+                auto parts = ggml_ext_chunk(ctx->ggml_ctx, kv, 2, 0);
+                qkv        = {q, parts[0], parts[1]};
+            }
+            auto out = ggml_ext_attention_ext(ctx, qkv[0], qkv[1], qkv[2], num_heads, mask, false, ctx->flash_attn_enabled);
+            return std::dynamic_pointer_cast<Linear>(blocks["proj"])->forward(ctx, out);
+        }
+    };
+
+    class PixArtBlock : public GGMLBlock {
+        int64_t dim;
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            ggml_type wtype             = get_type(prefix + "scale_shift_table", tensor_storage_map, GGML_TYPE_F32);
+            params["scale_shift_table"] = ggml_new_tensor_2d(ctx, wtype, dim, 6);
+        }
+
+    public:
+        PixArtBlock(int64_t dim, int64_t num_heads, int64_t context_dim, int64_t ffn_dim)
+            : dim(dim) {
+            blocks["attn"]       = std::make_shared<PixArtAttention>(dim, num_heads, dim, true);
+            blocks["cross_attn"] = std::make_shared<PixArtAttention>(dim, num_heads, context_dim, false);
+            blocks["mlp.fc1"]    = std::make_shared<Linear>(dim, ffn_dim);
+            blocks["mlp.fc2"]    = std::make_shared<Linear>(ffn_dim, dim);
+        }
+
+        static ggml_tensor* norm(ggml_context* ctx, ggml_tensor* x) {
+            return ggml_norm(ctx, x, 1e-6f);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* mod, ggml_tensor* context, ggml_tensor* context_mask) {
+            // x: [N, n_token, dim]
+            // mod: [N, 6 * dim], shared adaLN-single output
+            int64_t N = x->ne[2];
+
+            auto table = params["scale_shift_table"];
+            if (table->type != GGML_TYPE_F32) {
+                table = ggml_cast(ctx->ggml_ctx, table, GGML_TYPE_F32);
+            }
+            table   = ggml_reshape_3d(ctx->ggml_ctx, table, dim, 6, 1);
+            auto m  = ggml_add(ctx->ggml_ctx, ggml_reshape_3d(ctx->ggml_ctx, mod, dim, 6, N), table);
+            auto mv = ggml_ext_chunk(ctx->ggml_ctx, ggml_reshape_2d(ctx->ggml_ctx, ggml_ext_cont(ctx->ggml_ctx, m), dim * 6, N), 6, 0);
+
+            auto attn1 = std::dynamic_pointer_cast<PixArtAttention>(blocks["attn"]);
+            auto attn2 = std::dynamic_pointer_cast<PixArtAttention>(blocks["cross_attn"]);
+            auto proj  = std::dynamic_pointer_cast<Linear>(blocks["mlp.fc1"]);
+            auto fc2   = std::dynamic_pointer_cast<Linear>(blocks["mlp.fc2"]);
+
+            auto gate = [&](ggml_tensor* y, ggml_tensor* g) {
+                g = ggml_reshape_3d(ctx->ggml_ctx, g, dim, 1, N);
+                return ggml_add(ctx->ggml_ctx, x, ggml_mul(ctx->ggml_ctx, y, g));
+            };
+
+            auto h = modulate(ctx->ggml_ctx, norm(ctx->ggml_ctx, x), mv[0], mv[1]);
+            x      = gate(attn1->forward(ctx, h, h), mv[2]);
+            // ada_norm_single: no norm before cross-attention (PixArtMS.py)
+            x = ggml_add(ctx->ggml_ctx, x, attn2->forward(ctx, x, context, context_mask));
+            h = modulate(ctx->ggml_ctx, norm(ctx->ggml_ctx, x), mv[3], mv[4]);
+            h = proj->forward(ctx, h);
+            h = ggml_ext_gelu(ctx->ggml_ctx, h, true);
+            h = fc2->forward(ctx, h);
+            return gate(h, mv[5]);
+        }
+    };
+
+    class PixArtModel : public GGMLBlock {
+        PixArtConfig config;
+
+        void init_params(ggml_context* ctx,
+                         const String2TensorStorage& tensor_storage_map = {},
+                         const std::string prefix                       = "") override {
+            ggml_type wtype                         = get_type(prefix + "final_layer.scale_shift_table", tensor_storage_map, GGML_TYPE_F32);
+            params["final_layer.scale_shift_table"] = ggml_new_tensor_2d(ctx, wtype, config.hidden_size, 2);
+        }
+
+    public:
+        PixArtModel() = default;
+        PixArtModel(const PixArtConfig& config)
+            : config(config) {
+            blocks["x_embedder.proj"]       = std::make_shared<Conv2d>(config.in_channels,
+                                                                 config.hidden_size,
+                                                                 std::pair<int, int>{static_cast<int>(config.patch_size), static_cast<int>(config.patch_size)},
+                                                                 std::pair<int, int>{static_cast<int>(config.patch_size), static_cast<int>(config.patch_size)});
+            blocks["t_embedder"]            = std::make_shared<PixArtTimestepEmbedding>(ADALN_EMBED_DIM, config.hidden_size);
+            blocks["t_block.1"]             = std::make_shared<Linear>(config.hidden_size, 6 * config.hidden_size);
+            blocks["y_embedder.y_proj.fc1"] = std::make_shared<Linear>(config.caption_channels, config.hidden_size);
+            blocks["y_embedder.y_proj.fc2"] = std::make_shared<Linear>(config.hidden_size, config.cross_attention_dim);
+            for (int i = 0; i < config.num_layers; ++i) {
+                blocks["blocks." + std::to_string(i)] =
+                    std::make_shared<PixArtBlock>(config.hidden_size, config.num_heads, config.cross_attention_dim, config.ffn_dim);
+            }
+            blocks["final_layer.norm_final"] = std::make_shared<LayerNorm>(config.hidden_size, 1e-6f, false);
+            blocks["final_layer.linear"]     = std::make_shared<Linear>(config.hidden_size,
+                                                                    config.patch_size * config.patch_size * config.out_channels);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx,
+                             ggml_tensor* x,
+                             ggml_tensor* timesteps,
+                             ggml_tensor* context,
+                             ggml_tensor* pos_embed,
+                             ggml_tensor* context_mask) {
+            // x: [N, C, H, W] latent, context: [N, n_ctx, caption_channels]
+            int64_t W  = x->ne[0];
+            int64_t H  = x->ne[1];
+            int64_t N  = x->ne[3];
+            int64_t p  = config.patch_size;
+            int64_t wp = W / p;
+            int64_t hp = H / p;
+
+            auto h = std::dynamic_pointer_cast<Conv2d>(blocks["x_embedder.proj"])->forward(ctx, x);  // [N, hidden, hp, wp]
+            h      = ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, h, 1, 2, 0, 3));       // [N, hp, wp, hidden] -> [N, hp*wp, hidden]
+            h      = ggml_reshape_3d(ctx->ggml_ctx, h, config.hidden_size, wp * hp, N);              // [N, hp*wp, hidden]
+            h      = ggml_add(ctx->ggml_ctx, h, pos_embed);
+
+            auto t   = ggml_ext_timestep_embedding(ctx->ggml_ctx, timesteps, ADALN_EMBED_DIM, 10000);
+            auto emb = std::dynamic_pointer_cast<PixArtTimestepEmbedding>(blocks["t_embedder"])->forward(ctx, t);
+
+            auto mod = std::dynamic_pointer_cast<Linear>(blocks["t_block.1"])
+                           ->forward(ctx, ggml_silu(ctx->ggml_ctx, emb));  // [N, 6 * hidden]
+
+            auto ctx_emb = std::dynamic_pointer_cast<Linear>(blocks["y_embedder.y_proj.fc1"])->forward(ctx, context);
+            ctx_emb      = ggml_ext_gelu(ctx->ggml_ctx, ctx_emb, true);
+            ctx_emb      = std::dynamic_pointer_cast<Linear>(blocks["y_embedder.y_proj.fc2"])->forward(ctx, ctx_emb);
+
+            for (int i = 0; i < config.num_layers; ++i) {
+                auto block = std::dynamic_pointer_cast<PixArtBlock>(blocks["blocks." + std::to_string(i)]);
+                h          = block->forward(ctx, h, mod, ctx_emb, context_mask);
+                sd::ggml_graph_cut::mark_graph_cut(h, "pixart.blocks." + std::to_string(i), "h");
+            }
+
+            // scale_shift_table + emb -> (shift, scale) for the affine-free final norm
+            auto tail_table = params["final_layer.scale_shift_table"];
+            if (tail_table->type != GGML_TYPE_F32) {
+                tail_table = ggml_cast(ctx->ggml_ctx, tail_table, GGML_TYPE_F32);
+            }
+            auto ss    = ggml_add(ctx->ggml_ctx,
+                                  ggml_reshape_3d(ctx->ggml_ctx, tail_table, config.hidden_size, 2, 1),
+                                  ggml_reshape_3d(ctx->ggml_ctx, emb, config.hidden_size, 1, N));  // [2, hidden, N]
+            auto parts = ggml_ext_chunk(ctx->ggml_ctx,
+                                        ggml_reshape_2d(ctx->ggml_ctx, ggml_ext_cont(ctx->ggml_ctx, ss), config.hidden_size * 2, N),
+                                        2, 0);
+            h          = std::dynamic_pointer_cast<LayerNorm>(blocks["final_layer.norm_final"])->forward(ctx, h);
+            h          = modulate(ctx->ggml_ctx, h, parts[0], parts[1]);
+            h          = std::dynamic_pointer_cast<Linear>(blocks["final_layer.linear"])->forward(ctx, h);  // [N, hp*wp, p*p*out_ch]
+            h          = DiT::unpatchify(ctx->ggml_ctx, h, hp, wp, static_cast<int>(p), static_cast<int>(p), false);
+            return h;  // [N, out_channels, H, W]
+        }
+    };
+
+    struct PixArtRunner : public DiffusionModelRunner {
+        PixArtConfig config;
+        PixArtModel model;
+        std::vector<float> pos_vec;
+
+        PixArtRunner(ggml_backend_t backend,
+                     const String2TensorStorage& tensor_storage_map      = {},
+                     const std::string prefix                            = "",
+                     std::shared_ptr<RunnerWeightManager> weight_manager = nullptr,
+                     const char* model_args                              = nullptr)
+            : DiffusionModelRunner(backend, prefix, weight_manager),
+              config(PixArtConfig::detect_from_weights(tensor_storage_map, prefix)) {
+            for (const auto& [key, value] : parse_key_value_args(model_args, "model arg")) {
+                if (key == "pixart_pos_embed_base_size") {
+                    int parsed = 0;
+                    if (parse_strict_int(value, parsed)) {
+                        config.pos_embed_base_size = parsed;
+                    } else {
+                        LOG_WARN("ignoring invalid PixArt model arg '%s=%s'", key.c_str(), value.c_str());
+                    }
+                } else if (key == "pixart_interpolation_scale") {
+                    float parsed = 0.f;
+                    if (parse_strict_float(value, parsed)) {
+                        config.interpolation_scale = parsed;
+                    } else {
+                        LOG_WARN("ignoring invalid PixArt model arg '%s=%s'", key.c_str(), value.c_str());
+                    }
+                }
+            }
+            model = PixArtModel(config);
+            model.init(params_ctx, tensor_storage_map, prefix);
+        }
+
+        std::string get_desc() override {
+            return "pixart";
+        }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor,
+                                 const sd::Tensor<float>& timesteps_tensor,
+                                 const sd::Tensor<float>& context_tensor,
+                                 const sd::Tensor<float>& mask_tensor) {
+            ggml_cgraph* gf        = new_graph_custom(PIXART_GRAPH_SIZE);
+            ggml_tensor* x         = make_input(x_tensor);
+            ggml_tensor* timesteps = make_input(timesteps_tensor);
+            GGML_ASSERT(x->ne[3] == 1);
+            GGML_ASSERT(!context_tensor.empty());
+            ggml_tensor* context = make_input(context_tensor);
+
+            ggml_tensor* context_mask = nullptr;
+            if (!mask_tensor.empty()) {
+                // additive attention bias over context tokens: 0 keep / -inf discard
+                context_mask = ggml_reshape_4d(compute_ctx, make_input(mask_tensor), mask_tensor.shape()[0], 1, 1, 1);
+            }
+
+            int64_t W  = x->ne[0];
+            int64_t H  = x->ne[1];
+            int64_t wp = W / config.patch_size;
+            int64_t hp = H / config.patch_size;
+
+            pos_vec  = gen_2d_sincos_pos_embed(config.hidden_size, hp, wp,
+                                               config.pos_embed_base_size, config.interpolation_scale);
+            auto pos = ggml_new_tensor_3d(compute_ctx, GGML_TYPE_F32, config.hidden_size, wp * hp, 1);
+            set_backend_tensor_data(pos, pos_vec.data());
+
+            auto runner_ctx  = get_context();
+            ggml_tensor* out = model.forward(&runner_ctx, x, timesteps, context, pos, context_mask);
+            // learn_sigma: keep the noise prediction half of the output channels
+            out = ggml_ext_slice(compute_ctx, out, 2, 0, config.in_channels);
+            ggml_build_forward_expand(gf, out);
+            return gf;
+        }
+
+        sd::Tensor<float> compute(int n_threads,
+                                  const sd::Tensor<float>& x,
+                                  const sd::Tensor<float>& timesteps,
+                                  const sd::Tensor<float>& context,
+                                  const sd::Tensor<float>& context_mask) {
+            auto get_graph = [&]() -> ggml_cgraph* {
+                return build_graph(x, timesteps, context, context_mask);
+            };
+            return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
+        }
+
+        sd::Tensor<float> compute(int n_threads,
+                                  const DiffusionParams& diffusion_params) override {
+            GGML_ASSERT(diffusion_params.x != nullptr);
+            GGML_ASSERT(diffusion_params.timesteps != nullptr);
+            auto context     = tensor_or_empty(diffusion_params.context);
+            auto context_msk = tensor_or_empty(diffusion_params.y);
+            return compute(n_threads,
+                           *diffusion_params.x,
+                           *diffusion_params.timesteps,
+                           context,
+                           context_msk);
+        }
+    };
+}  // namespace PixArt
+
+#endif  // __SD_MODEL_DIFFUSION_PIXART_HPP__
diff --git a/src/model/diffusion/qwen_image.hpp b/src/model/diffusion/qwen_image.hpp
index 07364958f6..82d1730059 100644
--- a/src/model/diffusion/qwen_image.hpp
+++ b/src/model/diffusion/qwen_image.hpp
@@ -635,18 +635,16 @@ namespace Qwen {
                 ref_index_mode = Rope::RefIndexMode::DECREASE;
             }
 
-            pe_vec      = Rope::gen_qwen_image_pe(time_len,
-                                                  static_cast<int>(x->ne[1]),
-                                                  static_cast<int>(x->ne[0]),
-                                                  config.patch_size,
-                                                  batch_size,
-                                                  static_cast<int>(context->ne[1]),
-                                                  ref_latents,
-                                                  ref_index_mode,
-                                                  config.theta,
-                                                  circular_y_enabled,
-                                                  circular_x_enabled,
-                                                  config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_qwen_image_pe(time_len,
+                                                                 static_cast<int>(x->ne[1]),
+                                                                 static_cast<int>(x->ne[0]),
+                                                                 config.patch_size,
+                                                                 batch_size,
+                                                                 static_cast<int>(context->ne[1]),
+                                                                 ref_latents,
+                                                                 ref_index_mode,
+                                                                 config.theta,
+                                                                 config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
diff --git a/src/model/diffusion/qwen_image_2_1.hpp b/src/model/diffusion/qwen_image_2_1.hpp
index cb044053fe..1d37d6afeb 100644
--- a/src/model/diffusion/qwen_image_2_1.hpp
+++ b/src/model/diffusion/qwen_image_2_1.hpp
@@ -68,6 +68,7 @@ namespace Qwen {
         std::vector<QwenImage21Segment> segments;
         std::vector<std::vector<float>> positions;
         int64_t prefix_length = 0;
+        Rope::PositionLayout rope_layout;
 
         static QwenImage21Layout build(int64_t text_length,
                                        const sd::Tensor<int32_t>& image_slots,
@@ -82,6 +83,7 @@ namespace Qwen {
                 auto [height, width] = image_shapes[index];
                 int64_t start        = static_cast<int64_t>(layout.positions.size());
                 layout.segments.push_back({start, start + height * width, context_start, index});
+                layout.rope_layout.append_image(static_cast<int>(height), static_cast<int>(width));
                 for (int64_t h = 0; h < height; ++h) {
                     for (int64_t w = 0; w < width; ++w) {
                         layout.positions.push_back({static_cast<float>(position),
@@ -106,6 +108,7 @@ namespace Qwen {
                 } else {
                     int64_t start = static_cast<int64_t>(layout.positions.size());
                     layout.segments.push_back({start, start + i - begin, begin, -1});
+                    layout.rope_layout.append_tokens(i - begin);
                     for (int64_t j = begin; j < i; ++j, ++position) {
                         float p = static_cast<float>(position);
                         layout.positions.push_back({p, p, p});
@@ -121,6 +124,20 @@ namespace Qwen {
         }
     };
 
+    struct QwenImage21PrefixCache {
+        enum class Mode {
+            NONE,
+            STORE,
+            REUSE
+        };
+        Mode mode = Mode::NONE;
+        std::string name;
+        std::string cut_group;
+        int64_t prefix_length = 0;
+        ggml_type type        = GGML_TYPE_F32;
+        bool* flash_attn_used = nullptr;
+    };
+
     class QwenImage21ZeroCenterRMSNorm : public RMSNorm {
     public:
         using RMSNorm::RMSNorm;
@@ -160,27 +177,70 @@ namespace Qwen {
             }
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, const std::vector<QwenImage21Segment>& segments, const std::vector<ggml_tensor*>& masks) {
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* pe, const std::vector<QwenImage21Segment>& segments, const std::vector<ggml_tensor*>& masks, const QwenImage21PrefixCache& cache) {
             int64_t heads = x->ne[0] / dim_head;
             auto project  = [&](const char* name) {
                 auto h = std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, x);
                 return ggml_reshape_4d(ctx->ggml_ctx, h, dim_head, heads, x->ne[1], x->ne[2]);
             };
-            auto q              = project("to_q");
-            auto k              = project("to_k");
-            auto v              = project("to_v");
-            q                   = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_q"])->forward(ctx, q);
-            k                   = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_k"])->forward(ctx, k);
-            q                   = Rope::apply_rope(ctx->ggml_ctx, q, pe);
-            k                   = Rope::apply_rope(ctx->ggml_ctx, k, pe);
+            auto q = project("to_q");
+            auto k = project("to_k");
+            auto v = project("to_v");
+            q      = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_q"])->forward(ctx, q);
+            k      = std::dynamic_pointer_cast<RMSNorm>(blocks["norm_k"])->forward(ctx, k);
+            q      = Rope::apply_rope(ctx, q, pe);
+            k      = Rope::apply_rope(ctx, k, pe);
+            if (cache.mode == QwenImage21PrefixCache::Mode::STORE) {
+                // Preserve query-first attention evaluation while writing each layer's
+                // prefix before its full-sequence K/V can accumulate across layers.
+                ctx->expand_graph(q);
+                auto persist = [&](ggml_tensor* tensor, int axis, const char* name) {
+                    auto part = ggml_ext_slice(ctx->ggml_ctx, tensor, axis, 0, cache.prefix_length);
+                    // Pack the contiguous data into wider rows so quantization blocks
+                    // can exceed head_dim without padding or changing element order.
+                    part      = ggml_reshape_2d(ctx->ggml_ctx, part, x->ne[0], cache.prefix_length);
+                    auto copy = ggml_cast(ctx->ggml_ctx, part, cache.type);
+                    // Keep the copy in this layer's segment so graph cuts do not
+                    // retain or recompute the full-sequence K/V in the final segment.
+                    sd::ggml_graph_cut::mark_graph_cut(copy, cache.cut_group, name);
+                    ctx->persist_cache_tensor(cache.name + "." + name, copy);
+                };
+                persist(k, 1, "k");
+                persist(v, 2, "v");
+            }
+            auto attend = [&](ggml_tensor* aq, ggml_tensor* ak, ggml_tensor* av, ggml_tensor* mask) {
+                bool used_flash_attn = false;
+                auto out             = ggml_ext_attention_ext(ctx, aq, ak, av, heads, mask, true, ctx->flash_attn_enabled, 1.f, &used_flash_attn);
+                if (cache.flash_attn_used != nullptr) {
+                    *cache.flash_attn_used &= used_flash_attn;
+                }
+                return out;
+            };
             ggml_tensor* result = nullptr;
-            for (size_t i = 0; i < segments.size(); ++i) {
-                const auto& segment = segments[i];
-                auto sq             = ggml_ext_slice(ctx->ggml_ctx, q, 1, segment.start, segment.end);
-                auto sk             = ggml_ext_slice(ctx->ggml_ctx, k, 1, 0, segment.end);
-                auto sv             = ggml_ext_slice(ctx->ggml_ctx, v, 2, 0, segment.end);
-                auto out            = ggml_ext_attention_ext(ctx, sq, sk, sv, heads, masks[i], true, ctx->flash_attn_enabled);
-                result              = result == nullptr ? out : ggml_concat(ctx->ggml_ctx, result, out, 1);
+            if (cache.mode == QwenImage21PrefixCache::Mode::REUSE) {
+                auto prefix_k = ctx->load_cache_tensor(cache.name + ".k");
+                auto prefix_v = ctx->load_cache_tensor(cache.name + ".v");
+                GGML_ASSERT(prefix_k != nullptr && prefix_v != nullptr);
+                if (prefix_k->type != k->type) {
+                    prefix_k = ggml_cast(ctx->ggml_ctx, prefix_k, k->type);
+                }
+                if (prefix_v->type != v->type) {
+                    prefix_v = ggml_cast(ctx->ggml_ctx, prefix_v, v->type);
+                }
+                prefix_k = ggml_reshape_4d(ctx->ggml_ctx, prefix_k, dim_head, cache.prefix_length, heads, k->ne[3]);
+                prefix_v = ggml_reshape_4d(ctx->ggml_ctx, prefix_v, dim_head, heads, cache.prefix_length, v->ne[3]);
+                k        = ggml_concat(ctx->ggml_ctx, prefix_k, k, 1);
+                v        = ggml_concat(ctx->ggml_ctx, prefix_v, v, 2);
+                result   = attend(q, k, v, nullptr);
+            } else {
+                for (size_t i = 0; i < segments.size(); ++i) {
+                    const auto& segment = segments[i];
+                    auto sq             = ggml_ext_slice(ctx->ggml_ctx, q, 1, segment.start, segment.end);
+                    auto sk             = ggml_ext_slice(ctx->ggml_ctx, k, 1, 0, segment.end);
+                    auto sv             = ggml_ext_slice(ctx->ggml_ctx, v, 2, 0, segment.end);
+                    auto out            = attend(sq, sk, sv, masks[i]);
+                    result              = result == nullptr ? out : ggml_concat(ctx->ggml_ctx, result, out, 1);
+                }
             }
             auto to_out = std::dynamic_pointer_cast<Linear>(blocks["to_out.0"]);
             if (sd_backend_is(ctx->backend, "Vulkan") || sd_backend_is(ctx->backend, "ROCm")) {
@@ -219,13 +279,14 @@ namespace Qwen {
             return ggml_concat(ctx, prefix, target, 1);
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, const std::vector<ggml_tensor*>& modulation, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector<ggml_tensor*>& masks) {
-            auto h = std::dynamic_pointer_cast<LayerNorm>(blocks["img_norm1"])->forward(ctx, x);
-            h      = modulate(ctx->ggml_ctx, h, modulation[0], layout.prefix_length);
-            h      = std::dynamic_pointer_cast<QwenImage21Attention>(blocks["attn"])->forward(ctx, h, pe, layout.segments, masks);
-            x      = ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[1], layout.prefix_length, true));
-            h      = std::dynamic_pointer_cast<LayerNorm>(blocks["img_norm2"])->forward(ctx, x);
-            h      = modulate(ctx->ggml_ctx, h, modulation[2], layout.prefix_length);
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, const std::vector<ggml_tensor*>& modulation, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector<ggml_tensor*>& masks, const QwenImage21PrefixCache& cache) {
+            const int64_t prefix_length = cache.mode == QwenImage21PrefixCache::Mode::REUSE ? 0 : layout.prefix_length;
+            auto h                      = std::dynamic_pointer_cast<LayerNorm>(blocks["img_norm1"])->forward(ctx, x);
+            h                           = modulate(ctx->ggml_ctx, h, modulation[0], prefix_length);
+            h                           = std::dynamic_pointer_cast<QwenImage21Attention>(blocks["attn"])->forward(ctx, h, pe, layout.segments, masks, cache);
+            x                           = ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[1], prefix_length, true));
+            h                           = std::dynamic_pointer_cast<LayerNorm>(blocks["img_norm2"])->forward(ctx, x);
+            h                           = modulate(ctx->ggml_ctx, h, modulation[2], prefix_length);
             ggml_tensor* gate;
             auto fused = blocks.find("img_mlp.gate_up");
             if (fused != blocks.end()) {
@@ -239,7 +300,7 @@ namespace Qwen {
             }
             h = ggml_mul(ctx->ggml_ctx, h, ggml_silu(ctx->ggml_ctx, gate));
             h = std::dynamic_pointer_cast<Linear>(blocks["img_mlp.out"])->forward(ctx, h);
-            return ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[3], layout.prefix_length, true));
+            return ggml_add(ctx->ggml_ctx, x, modulate(ctx->ggml_ctx, h, modulation[3], prefix_length, true));
         }
     };
 
@@ -261,7 +322,7 @@ namespace Qwen {
             }
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* timestep, ggml_tensor* context, const std::vector<ggml_tensor*>& refs, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector<ggml_tensor*>& masks) {
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* timestep, ggml_tensor* context, const std::vector<ggml_tensor*>& refs, ggml_tensor* pe, const QwenImage21Layout& layout, const std::vector<ggml_tensor*>& masks, const QwenImage21PrefixCache& cache) {
             auto time = ggml_concat(ctx->ggml_ctx, timestep, ggml_ext_zeros_like(ctx->ggml_ctx, timestep), 0);
             // Runtime flow timesteps already use the [0, 1000] scale.
             time               = ggml_ext_timestep_embedding(ctx->ggml_ctx, time, 256, 10000, 1.f);
@@ -269,27 +330,37 @@ namespace Qwen {
             time               = ggml_silu(ctx->ggml_ctx, time);
             auto modulation    = std::dynamic_pointer_cast<Linear>(blocks["modulation.1"])->forward(ctx, time);
             auto mod           = ggml_ext_chunk(ctx->ggml_ctx, modulation, 4, 0);
-            auto text          = std::dynamic_pointer_cast<QwenImage21TextProjection>(blocks["txt_in"])->forward(ctx, context);
             auto img_in        = std::dynamic_pointer_cast<Linear>(blocks["img_in"]);
             ggml_tensor* joint = nullptr;
-            for (const auto& segment : layout.segments) {
-                ggml_tensor* h;
-                if (segment.image_index < 0) {
-                    h = ggml_ext_slice(ctx->ggml_ctx, text, 1, segment.context_start,
-                                       segment.context_start + segment.end - segment.start);
-                } else {
-                    auto image = segment.image_index == static_cast<int>(refs.size()) ? x : refs[segment.image_index];
-                    h          = img_in->forward(ctx, DiT::patchify(ctx->ggml_ctx, image, 1, 1));
+            if (cache.mode == QwenImage21PrefixCache::Mode::REUSE) {
+                joint = img_in->forward(ctx, DiT::patchify(ctx->ggml_ctx, x, 1, 1));
+            } else {
+                auto text = std::dynamic_pointer_cast<QwenImage21TextProjection>(blocks["txt_in"])->forward(ctx, context);
+                for (const auto& segment : layout.segments) {
+                    ggml_tensor* h;
+                    if (segment.image_index < 0) {
+                        h = ggml_ext_slice(ctx->ggml_ctx, text, 1, segment.context_start,
+                                           segment.context_start + segment.end - segment.start);
+                    } else {
+                        auto image = segment.image_index == static_cast<int>(refs.size()) ? x : refs[segment.image_index];
+                        h          = img_in->forward(ctx, DiT::patchify(ctx->ggml_ctx, image, 1, 1));
+                    }
+                    joint = joint == nullptr ? h : ggml_concat(ctx->ggml_ctx, joint, h, 1);
                 }
-                joint = joint == nullptr ? h : ggml_concat(ctx->ggml_ctx, joint, h, 1);
             }
             sd::ggml_graph_cut::mark_graph_cut(joint, "qwen_image_2_1.prelude", "joint");
             for (int i = 0; i < config.num_layers; ++i) {
-                auto block = std::dynamic_pointer_cast<QwenImage21TransformerBlock>(blocks["transformer_blocks." + std::to_string(i)]);
-                joint      = block->forward(ctx, joint, mod, pe, layout, masks);
-                sd::ggml_graph_cut::mark_graph_cut(joint, "qwen_image_2_1.transformer_blocks." + std::to_string(i), "joint");
+                const std::string layer = "transformer_blocks." + std::to_string(i);
+                auto layer_cache        = cache;
+                layer_cache.name        = cache.name + "." + std::to_string(i);
+                layer_cache.cut_group   = "qwen_image_2_1." + layer;
+                auto block              = std::dynamic_pointer_cast<QwenImage21TransformerBlock>(blocks[layer]);
+                joint                   = block->forward(ctx, joint, mod, pe, layout, masks, layer_cache);
+                sd::ggml_graph_cut::mark_graph_cut(joint, layer_cache.cut_group, "joint");
+            }
+            if (cache.mode != QwenImage21PrefixCache::Mode::REUSE) {
+                joint = ggml_ext_slice(ctx->ggml_ctx, joint, 1, layout.prefix_length, joint->ne[1]);
             }
-            joint      = ggml_ext_slice(ctx->ggml_ctx, joint, 1, layout.prefix_length, joint->ne[1]);
             auto scale = std::dynamic_pointer_cast<Linear>(blocks["norm_out.linear"])->forward(ctx, ggml_ext_chunk(ctx->ggml_ctx, time, 2, 1)[0]);
             joint      = std::dynamic_pointer_cast<LayerNorm>(blocks["norm_out.norm"])->forward(ctx, joint);
             joint      = ggml_mul(ctx->ggml_ctx, joint, ggml_scale_bias(ctx->ggml_ctx, scale, 1.f, 1.f));
@@ -303,11 +374,61 @@ namespace Qwen {
         QwenImage21Model model;
         std::vector<float> pe_data;
         std::vector<sd::Tensor<float>> mask_data;
+        ggml_type prefix_cache_type = GGML_TYPE_COUNT;
+        bool prefix_cache_enabled   = true;
+        bool prefix_cache_disabled  = false;
+        bool prefix_cache_auto_f32  = false;
 
-        QwenImage21Runner(ggml_backend_t backend, const String2TensorStorage& weights, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+        static bool supports_prefix_cache_type(ggml_type type) {
+            if (type == GGML_TYPE_F32) {
+                return true;
+            }
+            const auto* traits = ggml_get_type_traits(type);
+            if (traits->from_float_ref == nullptr || traits->to_float == nullptr) {
+                return false;
+            }
+            auto cpu = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);
+            if (cpu == nullptr) {
+                return false;
+            }
+            auto ctx = std::unique_ptr<ggml_context, decltype(&ggml_free)>(
+                ggml_init({3 * ggml_tensor_overhead(), nullptr, true}), ggml_free);
+            if (ctx == nullptr) {
+                return false;
+            }
+            // Some reference quantizers have no runtime copy support. Query the
+            // device through the registry so dynamically loaded CPU backends work.
+            auto source  = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, ggml_blck_size(type));
+            auto encoded = ggml_cast(ctx.get(), source, type);
+            auto decoded = ggml_cast(ctx.get(), encoded, GGML_TYPE_F32);
+            return ggml_backend_dev_supports_op(cpu, encoded) && ggml_backend_dev_supports_op(cpu, decoded);
+        }
+
+        QwenImage21Runner(ggml_backend_t backend, const String2TensorStorage& weights, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr, const char* model_args = nullptr)
             : DiffusionModelRunner(backend, prefix, weight_manager),
               config(QwenImage21Config::detect_from_weights(weights, prefix)),
               model(config) {
+            for (const auto& [key, value] : parse_key_value_args(model_args, "model arg")) {
+                if (key == "qwen_image_2_1_prefix_cache" && !parse_strict_bool(value, prefix_cache_enabled)) {
+                    LOG_WARN("ignoring invalid Qwen Image 2.1 model arg '%s=%s'", key.c_str(), value.c_str());
+                } else if (key == "qwen_image_2_1_prefix_cache_type") {
+                    if (value == "auto") {
+                        prefix_cache_type = GGML_TYPE_COUNT;
+                        continue;
+                    }
+                    const auto type = sd_type_to_ggml_type(str_to_sd_type(value.c_str()));
+                    if (type == GGML_TYPE_COUNT) {
+                        LOG_WARN("ignoring unknown Qwen Image 2.1 cache type '%s'", value.c_str());
+                    } else if (!supports_prefix_cache_type(type)) {
+                        LOG_WARN("ignoring Qwen Image 2.1 cache type '%s': runtime conversion to and from F32 is unavailable", value.c_str());
+                    } else if (config.hidden_size % ggml_blck_size(type) != 0) {
+                        LOG_WARN("ignoring Qwen Image 2.1 cache type '%s': block size %" PRId64 " does not divide hidden size %" PRId64,
+                                 value.c_str(), ggml_blck_size(type), config.hidden_size);
+                    } else {
+                        prefix_cache_type = type;
+                    }
+                }
+            }
             model.init(params_ctx, weights, prefix);
         }
 
@@ -317,6 +438,20 @@ namespace Qwen {
             model.get_param_tensors(tensors, prefix);
         }
 
+        bool has_prefix_cache(const QwenImage21PrefixCache& cache) {
+            for (int i = 0; i < config.num_layers; ++i) {
+                const auto name = cache.name + "." + std::to_string(i);
+                auto k          = get_cache_tensor_by_name(name + ".k");
+                auto v          = get_cache_tensor_by_name(name + ".v");
+                if (k == nullptr || v == nullptr || k->type != cache.type || v->type != cache.type ||
+                    k->ne[0] != config.hidden_size || k->ne[1] != cache.prefix_length || k->ne[2] != 1 || k->ne[3] != 1 ||
+                    v->ne[0] != config.hidden_size || v->ne[1] != cache.prefix_length || v->ne[2] != 1 || v->ne[3] != 1) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
         sd::Tensor<float> compute(int n_threads, const DiffusionParams& inputs) override {
             const auto& x       = tensor_or_empty(inputs.x);
             const auto& context = tensor_or_empty(inputs.context);
@@ -345,38 +480,113 @@ namespace Qwen {
                 LOG_ERROR("%s", error.what());
                 return {};
             }
-            pe_data = Rope::embed_nd(layout.positions, 1, 10000.f, config.axes_dim);
-            mask_data.clear();
-            for (const auto& segment : layout.segments) {
-                sd::Tensor<float> mask;
-                if (segment.image_index < 0) {
-                    mask = sd::Tensor<float>::zeros({segment.end, segment.end - segment.start});
-                    for (int64_t q = segment.start; q < segment.end; ++q) {
-                        for (int64_t k = q + 1; k < segment.end; ++k) {
-                            mask[k + segment.end * (q - segment.start)] = -INFINITY;
-                        }
-                    }
+            if (!runner_started()) {
+                prefix_cache_disabled = false;
+                prefix_cache_auto_f32 = false;
+            }
+            QwenImage21PrefixCache cache;
+            if (prefix_cache_enabled && !prefix_cache_disabled && extra != nullptr && extra->prefix_id != 0 && layout.prefix_length > 0) {
+                cache.name = "qwen_image_2_1.prefix." + std::to_string(extra->prefix_id) +
+                             ".circular." + std::to_string(circular_x_enabled) + std::to_string(circular_y_enabled);
+                cache.prefix_length = layout.prefix_length;
+                if (prefix_cache_type != GGML_TYPE_COUNT) {
+                    cache.type = prefix_cache_type;
+                } else if (!prefix_cache_auto_f32 && flash_attn_enabled && !sage_attn_enabled &&
+                           (attn_scale <= 0.f || attn_scale == 1.f)) {
+                    cache.type = GGML_TYPE_F16;
                 }
-                mask_data.push_back(std::move(mask));
-            }
-            auto build = [&]() {
-                auto graph = new_graph_custom(QWEN_IMAGE_GRAPH_SIZE * 2);
-                auto pe    = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2, layout.positions.size());
-                set_backend_tensor_data(pe, pe_data.data());
-                std::vector<ggml_tensor*> masks, ref_inputs;
-                for (const auto& mask : mask_data) {
-                    masks.push_back(mask.empty() ? nullptr : make_input(mask));
+                cache.mode = has_prefix_cache(cache) ? QwenImage21PrefixCache::Mode::REUSE : QwenImage21PrefixCache::Mode::STORE;
+            }
+            bool flash_attn_used = true;
+            auto run             = [&](const QwenImage21PrefixCache& active_cache) {
+                flash_attn_used    = true;
+                auto checked_cache = active_cache;
+                if (prefix_cache_type == GGML_TYPE_COUNT && active_cache.type == GGML_TYPE_F16) {
+                    checked_cache.flash_attn_used = &flash_attn_used;
                 }
-                for (const auto& ref : refs) {
-                    ref_inputs.push_back(make_input(ref));
+                const bool cached         = active_cache.mode == QwenImage21PrefixCache::Mode::REUSE;
+                const auto first_position = layout.positions.begin() + (cached ? layout.prefix_length : 0);
+                Rope::Embedding embedding;
+                embedding.ids.assign(first_position, layout.positions.end());
+                const size_t offset             = cached ? static_cast<size_t>(layout.prefix_length) : 0;
+                embedding.positions.token_count = embedding.ids.size();
+                for (auto region : layout.rope_layout.images) {
+                    if (region.begin >= offset) {
+                        region.begin -= offset;
+                        embedding.positions.images.push_back(region);
+                    }
                 }
-                auto ctx = get_context();
-                auto out = model.forward(&ctx, make_input(x), make_input(*inputs.timesteps), make_input(context),
-                                         ref_inputs, pe, layout, masks);
-                ggml_build_forward_expand(graph, out);
-                return graph;
+                embedding.values = Rope::embed_nd(embedding.ids, 1, 10000.f, config.axes_dim, embedding.layout, &embedding.frequencies);
+                pe_data          = finish_rope_pe(std::move(embedding));
+                mask_data.clear();
+                if (!cached) {
+                    for (const auto& segment : layout.segments) {
+                        sd::Tensor<float> mask;
+                        if (segment.image_index < 0) {
+                            mask = sd::Tensor<float>::zeros({segment.end, segment.end - segment.start});
+                            for (int64_t q = segment.start; q < segment.end; ++q) {
+                                for (int64_t k = q + 1; k < segment.end; ++k) {
+                                    mask[k + segment.end * (q - segment.start)] = -INFINITY;
+                                }
+                            }
+                        }
+                        mask_data.push_back(std::move(mask));
+                    }
+                }
+                auto build = [&]() {
+                    auto graph = new_graph_custom(QWEN_IMAGE_GRAPH_SIZE * 2);
+                    auto pe    = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.head_dim / 2,
+                                                                layout.positions.size() - (cached ? layout.prefix_length : 0));
+                    set_backend_tensor_data(pe, pe_data.data());
+                    std::vector<ggml_tensor*> masks, ref_inputs;
+                    for (const auto& mask : mask_data) {
+                        masks.push_back(mask.empty() ? nullptr : make_input(mask));
+                    }
+                    if (!cached) {
+                        for (const auto& ref : refs) {
+                            ref_inputs.push_back(make_input(ref));
+                        }
+                    }
+                    auto ctx = get_context(graph);
+                    auto out = model.forward(&ctx, make_input(x), make_input(*inputs.timesteps), cached ? nullptr : make_input(context),
+                                                         ref_inputs, pe, layout, masks, checked_cache);
+                    if (!flash_attn_used) {
+                        return static_cast<ggml_cgraph*>(nullptr);
+                    }
+                    ggml_build_forward_expand(graph, out);
+                    return graph;
+                };
+                return restore_trailing_singleton_dims(GGMLRunner::compute(build, n_threads, false), x.dim());
             };
-            return restore_trailing_singleton_dims(GGMLRunner::compute(build, n_threads, false), x.dim());
+            auto result = run(cache);
+            if (result.empty() && !flash_attn_used) {
+                // Casting an F16 cache back to F32 cannot recover its original values.
+                // Recompute the prefix before executing a graph that falls back from FA.
+                free_cache_ctx_and_buffer();
+                prefix_cache_auto_f32 = true;
+                cache.type            = GGML_TYPE_F32;
+                cache.mode            = QwenImage21PrefixCache::Mode::STORE;
+                LOG_DEBUG("Qwen Image 2.1: Flash Attention unavailable; using F32 prefix caching for this sampling run");
+                result = run(cache);
+            }
+            if (result.empty() && last_compute_status() == GGML_STATUS_ALLOC_FAILED &&
+                (cache.mode != QwenImage21PrefixCache::Mode::NONE || !cache_.empty())) {
+                // The failed graph has ended before persistent inputs are released.
+                free_cache_ctx_and_buffer();
+                prefix_cache_disabled = true;
+                LOG_WARN("Qwen Image 2.1: insufficient memory for prefix caching; retrying without it for this sampling run");
+                return run(QwenImage21PrefixCache{});
+            }
+            if (!result.empty() && cache.mode == QwenImage21PrefixCache::Mode::STORE) {
+                if (!has_prefix_cache(cache)) {
+                    free_cache_ctx_and_buffer();
+                    prefix_cache_disabled = true;
+                    LOG_WARN("Qwen Image 2.1: incomplete prefix cache; disabling it for this sampling run");
+                } else {
+                    LOG_DEBUG("Qwen Image 2.1: cached prefix %" PRIu64 " (%" PRId64 " tokens, %s)", extra->prefix_id, layout.prefix_length, ggml_type_name(cache.type));
+                }
+            }
+            return result;
         }
     };
 }
diff --git a/src/model/diffusion/sensenova_u1.h b/src/model/diffusion/sensenova_u1.h
index 22b789fac5..1702ed8546 100644
--- a/src/model/diffusion/sensenova_u1.h
+++ b/src/model/diffusion/sensenova_u1.h
@@ -442,16 +442,9 @@ namespace SenseNovaU1 {
                 k = ggml_concat(ctx->ggml_ctx, prefix_k, k, 2);
                 v = ggml_concat(ctx->ggml_ctx, prefix_v, v, 2);
             } else {
-                // Keep dedicated graph outputs alive until the runner copies them
-                // into its persistent cache buffer after graph execution.
-                auto cache_k = ggml_dup_tensor(ctx->ggml_ctx, k);
-                cache_k      = ggml_cpy(ctx->ggml_ctx, k, cache_k);
-                ggml_set_output(cache_k);
-                auto cache_v = ggml_dup_tensor(ctx->ggml_ctx, v);
-                cache_v      = ggml_cpy(ctx->ggml_ctx, v, cache_v);
-                ggml_set_output(cache_v);
-                ctx->persist_cache_tensor(layer_cache + ".k", cache_k);
-                ctx->persist_cache_tensor(layer_cache + ".v", cache_v);
+                ctx->expand_graph(q);
+                ctx->persist_cache_tensor(layer_cache + ".k", k);
+                ctx->persist_cache_tensor(layer_cache + ".v", v);
             }
 
             q = ggml_cont(ctx->ggml_ctx,
@@ -687,7 +680,7 @@ namespace SenseNovaU1 {
             ggml_set_name(attention_mask, "snu15.prefix.attention_mask");
             set_backend_tensor_data(attention_mask, attention_mask_vec.data());
 
-            auto runner_ctx = get_context();
+            auto runner_ctx = get_context(graph);
             auto text_model = model.text_model();
             auto hidden     = text_model->embed(&runner_ctx, ids);
             hidden          = text_model->forward(&runner_ctx,
diff --git a/src/model/diffusion/unet.hpp b/src/model/diffusion/unet.hpp
index 1c8ddf4e60..7c0ff09506 100644
--- a/src/model/diffusion/unet.hpp
+++ b/src/model/diffusion/unet.hpp
@@ -587,7 +587,9 @@ class UnetModelBlock : public GGMLBlock {
             label_emb      = ggml_silu_inplace(ctx->ggml_ctx, label_emb);
             label_emb      = label_embed_2->forward(ctx, label_emb);  // [N, time_embed_dim]
 
-            emb = ggml_add(ctx->ggml_ctx, emb, label_emb);  // [N, time_embed_dim]
+            emb = label_emb->ne[1] > emb->ne[1]
+                      ? ggml_add(ctx->ggml_ctx, label_emb, emb)
+                      : ggml_add(ctx->ggml_ctx, emb, label_emb);  // [N, time_embed_dim]
         }
         // sd::ggml_graph_cut::mark_graph_cut(emb, "unet.prelude", "emb");
 
diff --git a/src/model/diffusion/z_image.hpp b/src/model/diffusion/z_image.hpp
index a1012cad84..ed2644047e 100644
--- a/src/model/diffusion/z_image.hpp
+++ b/src/model/diffusion/z_image.hpp
@@ -140,7 +140,8 @@ namespace ZImage {
                        int64_t num_kv_heads,
                        bool qk_norm,
                        bool norm_elementwise_affine = true,
-                       bool split_qkv               = false)
+                       bool split_qkv               = false,
+                       float qk_norm_eps            = 1e-6f)
             : head_dim(head_dim), num_heads(num_heads), num_kv_heads(num_kv_heads), qk_norm(qk_norm), split_qkv(split_qkv) {
             float scale = 1.f;
             if (split_qkv) {
@@ -153,8 +154,8 @@ namespace ZImage {
                 blocks["out"] = std::make_shared<Linear>(num_heads * head_dim, hidden_size, false, false, false, scale);
             }
             if (qk_norm) {
-                blocks["q_norm"] = std::make_shared<RMSNorm>(head_dim, 1e-06f, norm_elementwise_affine);
-                blocks["k_norm"] = std::make_shared<RMSNorm>(head_dim, 1e-06f, norm_elementwise_affine);
+                blocks["q_norm"] = std::make_shared<RMSNorm>(head_dim, qk_norm_eps, norm_elementwise_affine);
+                blocks["k_norm"] = std::make_shared<RMSNorm>(head_dim, qk_norm_eps, norm_elementwise_affine);
             }
         }
 
@@ -167,13 +168,16 @@ namespace ZImage {
             int64_t N       = x->ne[2];
             auto out_proj   = std::dynamic_pointer_cast<Linear>(blocks[split_qkv ? "to_out.0" : "out"]);
 
+            if (out_proj->has_quantized_weight()) {
+                out_proj->set_scale(1.f / 16.f);
+            }
+
             if (split_qkv) {
                 auto q_proj = std::dynamic_pointer_cast<Linear>(blocks["to_q"]);
                 auto k_proj = std::dynamic_pointer_cast<Linear>(blocks["to_k"]);
                 auto v_proj = std::dynamic_pointer_cast<Linear>(blocks["to_v"]);
 
                 if (sd_backend_is(ctx->backend, "ROCm")) {
-                    out_proj->set_scale(1.f / 16.f);
                     out_proj->set_force_prec_f32(true);
                     q_proj->set_force_prec_f32(true);
                     k_proj->set_force_prec_f32(true);
@@ -196,7 +200,6 @@ namespace ZImage {
             auto qkv_proj = std::dynamic_pointer_cast<Linear>(blocks["qkv"]);
 
             if (sd_backend_is(ctx->backend, "ROCm")) {
-                out_proj->set_scale(1.f / 16.f);
                 out_proj->set_force_prec_f32(true);
                 qkv_proj->set_force_prec_f32(true);
             }
@@ -318,9 +321,10 @@ namespace ZImage {
                               bool qk_norm,
                               bool modulation              = true,
                               bool norm_elementwise_affine = true,
-                              bool split_qkv               = false)
+                              bool split_qkv               = false,
+                              float qk_norm_eps            = 1e-6f)
             : modulation(modulation) {
-            blocks["attention"]       = std::make_shared<JointAttention>(hidden_size, head_dim, num_heads, num_kv_heads, qk_norm, norm_elementwise_affine, split_qkv);
+            blocks["attention"]       = std::make_shared<JointAttention>(hidden_size, head_dim, num_heads, num_kv_heads, qk_norm, norm_elementwise_affine, split_qkv, qk_norm_eps);
             blocks["feed_forward"]    = std::make_shared<FeedForward>(hidden_size, hidden_size, multiple_of, ffn_dim_multiplier);
             blocks["attention_norm1"] = std::make_shared<RMSNorm>(hidden_size, norm_eps, norm_elementwise_affine);
             blocks["ffn_norm1"]       = std::make_shared<RMSNorm>(hidden_size, norm_eps, norm_elementwise_affine);
@@ -642,18 +646,16 @@ namespace ZImage {
                 ref_latents.push_back(make_input(ref_latent_tensor));
             }
 
-            pe_vec      = Rope::gen_z_image_pe(static_cast<int>(x->ne[1]),
-                                               static_cast<int>(x->ne[0]),
-                                               config.patch_size,
-                                               static_cast<int>(x->ne[3]),
-                                               static_cast<int>(context->ne[1]),
-                                               SEQ_MULTI_OF,
-                                               ref_latents,
-                                               ref_index_mode,
-                                               config.theta,
-                                               circular_y_enabled,
-                                               circular_x_enabled,
-                                               config.axes_dim);
+            pe_vec      = finish_rope_pe(Rope::gen_z_image_pe(static_cast<int>(x->ne[1]),
+                                                              static_cast<int>(x->ne[0]),
+                                                              config.patch_size,
+                                                              static_cast<int>(x->ne[3]),
+                                                              static_cast<int>(context->ne[1]),
+                                                              SEQ_MULTI_OF,
+                                                              ref_latents,
+                                                              ref_index_mode,
+                                                              config.theta,
+                                                              config.axes_dim));
             int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
             // LOG_VERBOSE("pos_len %d", pos_len);
             auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
diff --git a/src/model/diffusion/z_image_l2p.hpp b/src/model/diffusion/z_image_l2p.hpp
new file mode 100644
index 0000000000..0eb0b14d3d
--- /dev/null
+++ b/src/model/diffusion/z_image_l2p.hpp
@@ -0,0 +1,247 @@
+#ifndef __SD_MODEL_DIFFUSION_Z_IMAGE_L2P_HPP__
+#define __SD_MODEL_DIFFUSION_Z_IMAGE_L2P_HPP__
+
+#include <cmath>
+
+#include "z_image.hpp"
+
+// Ref: https://github.com/TencentYoutuResearch/T2I-L2P/blob/main/diffsynth/models/z_image_dit_L2P.py
+
+namespace ZImageL2P {
+    struct ZImageL2PConfig : ZImage::ZImageConfig {
+        static ZImageL2PConfig detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) {
+            ZImageL2PConfig config;
+            static_cast<ZImage::ZImageConfig&>(config) = ZImage::ZImageConfig::detect_from_weights(tensors, prefix);
+            config.in_channels                         = 3;
+            config.out_channels                        = 3;
+            config.patch_size                          = 16;
+            auto x_embedder                            = tensors.find(prefix + ".x_embedder.weight");
+            if (x_embedder != tensors.end()) {
+                config.patch_size = static_cast<int>(std::lround(std::sqrt(static_cast<double>(x_embedder->second.ne[0] / config.in_channels))));
+            }
+            // L2P ships split to_q/to_k/to_v, which name conversion maps to qkv.weight plus
+            // .1/.2 parts; the base detection only sees the q part and undercounts kv heads.
+            auto k_part = tensors.find(prefix + ".layers.0.attention.qkv.weight.1");
+            if (k_part != tensors.end()) {
+                config.num_kv_heads = k_part->second.ne[1] / config.head_dim;
+            }
+            LOG_VERBOSE("z_image_l2p: patch_size = %d, in_channels = %" PRId64 ", num_heads = %" PRId64 ", num_kv_heads = %" PRId64,
+                        config.patch_size,
+                        config.in_channels,
+                        config.num_heads,
+                        config.num_kv_heads);
+            return config;
+        }
+    };
+
+    // U-Net over the full-resolution noisy image. Its bottleneck sits at the DiT token
+    // grid, so the number of pooling levels is tied to patch_size == 16.
+    class LocalDecoder : public GGMLBlock {
+        static constexpr int LEVELS                  = 4;
+        static constexpr int64_t CHANNELS[LEVELS]    = {64, 128, 256, 512};
+        static constexpr int64_t BOTTLENECK_CHANNELS = 512;
+
+    public:
+        LocalDecoder(int64_t in_channels, int64_t cond_channels) {
+            int64_t prev = in_channels;
+            for (int i = 0; i < LEVELS; ++i) {
+                blocks["enc" + std::to_string(i + 1) + ".0"] = std::make_shared<Conv2d>(prev, CHANNELS[i], std::pair{3, 3}, std::pair{1, 1}, std::pair{1, 1});
+                prev                                         = CHANNELS[i];
+            }
+            blocks["bottleneck.0"] = std::make_shared<Conv2d>(prev + cond_channels, BOTTLENECK_CHANNELS, std::pair{1, 1});
+            prev                   = BOTTLENECK_CHANNELS;
+            for (int i = LEVELS - 1; i >= 0; --i) {
+                const std::string level      = std::to_string(i + 1);
+                const int64_t out            = i == 0 ? CHANNELS[0] : CHANNELS[i - 1];
+                blocks["up" + level + ".1"]  = std::make_shared<Conv2d>(prev, prev, std::pair{3, 3}, std::pair{1, 1}, std::pair{1, 1});
+                blocks["dec" + level + ".0"] = std::make_shared<Conv2d>(prev + CHANNELS[i], out, std::pair{3, 3}, std::pair{1, 1}, std::pair{1, 1});
+                prev                         = out;
+            }
+            blocks["out_conv"] = std::make_shared<Conv2d>(prev, in_channels, std::pair{1, 1});
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* cond) {
+            // x: [N, C, H, W]
+            // cond: [N, cond_channels, H / 16, W / 16]
+            // return: [N, C, H, W]
+            auto gctx = ctx->ggml_ctx;
+            auto conv = [&](const std::string& name, ggml_tensor* h) {
+                return std::dynamic_pointer_cast<Conv2d>(blocks[name])->forward(ctx, h);
+            };
+
+            std::vector<ggml_tensor*> skips;
+            auto h = x;
+            for (int i = 0; i < LEVELS; ++i) {
+                h = ggml_silu(gctx, conv("enc" + std::to_string(i + 1) + ".0", h));
+                skips.push_back(h);
+                h = ggml_pool_2d(gctx, h, GGML_OP_POOL_MAX, 2, 2, 2, 2, 0, 0);
+            }
+
+            h = ggml_concat(gctx, h, cond, 2);
+            h = ggml_silu(gctx, conv("bottleneck.0", h));
+
+            for (int i = LEVELS - 1; i >= 0; --i) {
+                const std::string level = std::to_string(i + 1);
+                h                       = ggml_upscale(gctx, h, 2, GGML_SCALE_MODE_NEAREST);
+                h                       = conv("up" + level + ".1", h);
+                h                       = ggml_concat(gctx, h, skips[i], 2);
+                h                       = ggml_silu(gctx, conv("dec" + level + ".0", h));
+            }
+            return conv("out_conv", h);
+        }
+    };
+
+    class ZImageL2PModel : public GGMLBlock {
+        ZImageL2PConfig config;
+
+        void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
+            params["cap_pad_token"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, config.hidden_size);
+            params["x_pad_token"]   = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, config.hidden_size);
+        }
+
+    public:
+        explicit ZImageL2PModel(const ZImageL2PConfig& config)
+            : config(config) {
+            blocks["x_embedder"]     = std::make_shared<Linear>(config.patch_size * config.patch_size * config.in_channels, config.hidden_size);
+            blocks["t_embedder"]     = std::make_shared<TimestepEmbedder>(MIN(config.hidden_size, 1024), 256, 256);
+            blocks["cap_embedder.0"] = std::make_shared<RMSNorm>(config.cap_feat_dim, config.norm_eps);
+            blocks["cap_embedder.1"] = std::make_shared<Linear>(config.cap_feat_dim, config.hidden_size);
+            auto add_blocks          = [&](const std::string& prefix, int64_t count, bool modulation) {
+                for (int64_t i = 0; i < count; ++i) {
+                    blocks[prefix + std::to_string(i)] = std::make_shared<ZImage::JointTransformerBlock>(
+                        static_cast<int>(i), config.hidden_size, config.head_dim, config.num_heads,
+                        config.num_kv_heads, config.multiple_of, config.ffn_dim_multiplier,
+                        config.norm_eps, config.qk_norm, modulation, true, false, 1e-5f);
+                }
+            };
+            add_blocks("noise_refiner.", config.num_refiner_layers, true);
+            add_blocks("context_refiner.", config.num_refiner_layers, false);
+            add_blocks("layers.", config.num_layers, true);
+            blocks["local_decoder"] = std::make_shared<LocalDecoder>(config.in_channels, config.hidden_size);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* timestep, ggml_tensor* context, ggml_tensor* pe) {
+            // x: [N, C, H, W], H and W are multiples of patch_size
+            // context: [N, L, cap_feat_dim]
+            // return: [N, C, H, W]
+            auto gctx           = ctx->ggml_ctx;
+            const int64_t W     = x->ne[0];
+            const int64_t H     = x->ne[1];
+            const int64_t N     = x->ne[3];
+            const int64_t w     = W / config.patch_size;
+            const int64_t h     = H / config.patch_size;
+            const int64_t n_img = w * h;
+            const int64_t n_txt = context->ne[1];
+
+            auto img = DiT::patchify(gctx, x, config.patch_size, config.patch_size, false);
+            img      = std::dynamic_pointer_cast<Linear>(blocks["x_embedder"])->forward(ctx, img);
+            auto txt = std::dynamic_pointer_cast<RMSNorm>(blocks["cap_embedder.0"])->forward(ctx, context);
+            txt      = std::dynamic_pointer_cast<Linear>(blocks["cap_embedder.1"])->forward(ctx, txt);
+            auto t   = std::dynamic_pointer_cast<TimestepEmbedder>(blocks["t_embedder"])->forward(ctx, timestep);
+            sd::ggml_graph_cut::mark_graph_cut(txt, "z_image_l2p.prelude", "txt");
+            sd::ggml_graph_cut::mark_graph_cut(img, "z_image_l2p.prelude", "img");
+            sd::ggml_graph_cut::mark_graph_cut(t, "z_image_l2p.prelude", "t_emb");
+
+            const int64_t n_txt_pad = Rope::bound_mod(static_cast<int>(n_txt), ZImage::SEQ_MULTI_OF);
+            if (n_txt_pad > 0) {
+                auto pad = params["cap_pad_token"];
+                txt      = ggml_concat(gctx, txt, ggml_repeat_4d(gctx, pad, pad->ne[0], n_txt_pad, N, 1), 1);
+            }
+            const int64_t n_img_pad = Rope::bound_mod(static_cast<int>(n_img), ZImage::SEQ_MULTI_OF);
+            if (n_img_pad > 0) {
+                auto pad = params["x_pad_token"];
+                img      = ggml_concat(gctx, img, ggml_repeat_4d(gctx, pad, pad->ne[0], n_img_pad, N, 1), 1);
+            }
+            GGML_ASSERT(txt->ne[1] + img->ne[1] == pe->ne[3]);
+
+            auto txt_pe = ggml_ext_slice(gctx, pe, 3, 0, txt->ne[1]);
+            auto img_pe = ggml_ext_slice(gctx, pe, 3, txt->ne[1], pe->ne[3]);
+            for (int64_t i = 0; i < config.num_refiner_layers; ++i) {
+                txt = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["context_refiner." + std::to_string(i)])->forward(ctx, txt, txt_pe);
+                sd::ggml_graph_cut::mark_graph_cut(txt, "z_image_l2p.context_refiner." + std::to_string(i), "txt");
+            }
+            for (int64_t i = 0; i < config.num_refiner_layers; ++i) {
+                img = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["noise_refiner." + std::to_string(i)])->forward(ctx, img, img_pe, nullptr, t);
+                sd::ggml_graph_cut::mark_graph_cut(img, "z_image_l2p.noise_refiner." + std::to_string(i), "img");
+            }
+
+            auto txt_img = ggml_concat(gctx, txt, img, 1);
+            for (int64_t i = 0; i < config.num_layers; ++i) {
+                txt_img = std::dynamic_pointer_cast<ZImage::JointTransformerBlock>(blocks["layers." + std::to_string(i)])->forward(ctx, txt_img, pe, nullptr, t);
+                sd::ggml_graph_cut::mark_graph_cut(txt_img, "z_image_l2p.layers." + std::to_string(i), "txt_img");
+            }
+
+            // The local decoder consumes the raw last-layer hidden states: there is no final
+            // norm or adaLN modulation in front of it, unlike Z-Image's final_layer.
+            const int64_t img_start = n_txt + n_txt_pad;
+            auto feat               = ggml_ext_slice(gctx, txt_img, 1, img_start, img_start + n_img);                           // [N, h*w, hidden_size]
+            feat                    = ggml_reshape_4d(gctx, feat, config.hidden_size, w, h, N);                                 // [N, h, w, hidden_size]
+            feat                    = ggml_cont(gctx, ggml_permute(gctx, feat, 2, 0, 1, 3));                                    // [N, hidden_size, h, w]
+            auto out                = std::dynamic_pointer_cast<LocalDecoder>(blocks["local_decoder"])->forward(ctx, x, feat);  // [N, C, H, W]
+            return ggml_ext_scale(gctx, out, -1.f);
+        }
+    };
+
+    struct ZImageL2PRunner : public DiffusionModelRunner {
+        ZImageL2PConfig config;
+        ZImageL2PModel model;
+        std::vector<float> pe_vec;
+
+        ZImageL2PRunner(ggml_backend_t backend,
+                        const String2TensorStorage& tensors,
+                        const std::string& prefix,
+                        std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : DiffusionModelRunner(backend, prefix, weight_manager),
+              config(ZImageL2PConfig::detect_from_weights(tensors, prefix)),
+              model(config) {
+            model.init(params_ctx, tensors, prefix);
+        }
+
+        std::string get_desc() override { return "z_image_l2p"; }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) override {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        sd::Tensor<float> compute(int n_threads, const DiffusionParams& inputs) override {
+            GGML_ASSERT(inputs.x != nullptr);
+            GGML_ASSERT(inputs.timesteps != nullptr);
+            if (inputs.ref_latents != nullptr && !inputs.ref_latents->empty()) {
+                LOG_ERROR("Z-Image L2P reference-image conditioning is not supported");
+                return {};
+            }
+            if (inputs.context == nullptr || inputs.context->empty()) {
+                LOG_ERROR("Z-Image L2P requires a text condition");
+                return {};
+            }
+            auto graph = [&]() {
+                auto gf      = new_graph_custom(ZImage::Z_IMAGE_GRAPH_SIZE);
+                auto x       = make_input(*inputs.x);
+                auto t       = make_input(*inputs.timesteps);
+                auto context = make_input(*inputs.context);
+                GGML_ASSERT(x->ne[3] == 1);
+                GGML_ASSERT(x->ne[0] % config.patch_size == 0 && x->ne[1] % config.patch_size == 0);
+                pe_vec      = finish_rope_pe(Rope::gen_z_image_pe(static_cast<int>(x->ne[1]),
+                                                                  static_cast<int>(x->ne[0]),
+                                                                  config.patch_size,
+                                                                  static_cast<int>(x->ne[3]),
+                                                                  static_cast<int>(context->ne[1]),
+                                                                  ZImage::SEQ_MULTI_OF,
+                                                                  {},
+                                                                  Rope::RefIndexMode::FIXED,
+                                                                  config.theta,
+                                                                  config.axes_dim));
+                int pos_len = static_cast<int>(pe_vec.size() / config.axes_dim_sum / 2);
+                auto pe     = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, config.axes_dim_sum / 2, pos_len);
+                set_backend_tensor_data(pe, pe_vec.data());
+                auto ctx = get_context();
+                auto out = model.forward(&ctx, x, t, context, pe);
+                ggml_build_forward_expand(gf, out);
+                return gf;
+            };
+            return restore_trailing_singleton_dims(GGMLRunner::compute(graph, n_threads, false), inputs.x->dim());
+        }
+    };
+}  // namespace ZImageL2P
+
+#endif  // __SD_MODEL_DIFFUSION_Z_IMAGE_L2P_HPP__
diff --git a/src/model/te/llm.hpp b/src/model/te/llm.hpp
index da59cfab61..8ea918025e 100644
--- a/src/model/te/llm.hpp
+++ b/src/model/te/llm.hpp
@@ -43,6 +43,7 @@ namespace LLM {
         QWEN2_5_VL,
         QWEN3,
         QWEN3_VL,
+        QWEN3_5,
         MISTRAL_SMALL_3_2,
         MINISTRAL_3_3B,
         GEMMA3_12B,
@@ -50,6 +51,8 @@ namespace LLM {
         GEMMA4_12B,
         GPT_OSS_20B,
         LLADA2_MOE,
+        BAILING_MOE,
+        QWEN2,
         ARCH_COUNT,
     };
 
@@ -57,6 +60,7 @@ namespace LLM {
         "qwen2.5vl",
         "qwen3",
         "qwen3vl",
+        "qwen3.5",
         "mistral_small3.2",
         "ministral3.3b",
         "gemma3_12b",
@@ -64,6 +68,8 @@ namespace LLM {
         "gemma4_12b",
         "gpt_oss_20b",
         "llada2_moe",
+        "bailing_moe",
+        "qwen2",
     };
 
     enum class MLPActivation {
@@ -117,6 +123,7 @@ namespace LLM {
         bool qk_norm                    = false;
         bool final_norm                 = true;
         bool rms_norm_add               = false;
+        bool qwen35_gguf                = false;
         bool normalize_input            = false;
         int64_t vocab_size              = 152064;
         int64_t max_position_embeddings = 128000;
@@ -130,6 +137,11 @@ namespace LLM {
         bool qkv_fused              = false;
         bool bidirectional          = false;
         float partial_rotary        = 1.f;
+        int linear_head_dim         = 128;
+        int linear_key_heads        = 16;
+        int linear_value_heads      = 32;
+        int linear_conv_kernel      = 4;
+        std::set<int> linear_attention_layers;
 
         // DeepSeek-V3-style grouped-sigmoid MoE routing (LLaDA2)
         int64_t moe_intermediate_size = 0;
@@ -163,6 +175,19 @@ namespace LLM {
                 config.num_kv_heads = 8;
                 config.qkv_bias     = false;
                 config.rms_norm_eps = 1e-5f;
+            } else if (arch == LLMArch::QWEN3_5) {
+                config.hidden_size             = 4096;
+                config.intermediate_size       = 12288;
+                config.head_dim                = 256;
+                config.num_heads               = 16;
+                config.num_kv_heads            = 4;
+                config.qkv_bias                = false;
+                config.qk_norm                 = true;
+                config.rms_norm_add            = true;
+                config.partial_rotary          = 0.25f;
+                config.rope_thetas             = {10000000.f};
+                config.max_position_embeddings = 262144;
+                config.vocab_size              = 248320;
             } else if (arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) {
                 config.head_dim     = 128;
                 config.num_heads    = 32;
@@ -225,7 +250,7 @@ namespace LLM {
                 config.intermediate_size       = 9216;
                 config.num_layers              = 26;
                 config.vocab_size              = 256000;
-            } else if (arch == LLMArch::LLADA2_MOE) {
+            } else if (arch == LLMArch::LLADA2_MOE || arch == LLMArch::BAILING_MOE) {
                 config.head_dim                = 128;
                 config.num_heads               = 16;
                 config.num_kv_heads            = 4;
@@ -240,7 +265,7 @@ namespace LLM {
                 config.max_position_embeddings = 16384;
                 config.rope_thetas             = {600000.f};
                 config.qkv_fused               = true;
-                config.bidirectional           = true;
+                config.bidirectional           = arch == LLMArch::LLADA2_MOE;
                 config.partial_rotary          = 0.5f;
                 config.num_experts             = 256;
                 config.num_experts_per_tok     = 8;
@@ -250,6 +275,18 @@ namespace LLM {
                 config.n_group                 = 8;
                 config.topk_group              = 4;
                 config.routed_scaling_factor   = 2.5f;
+                if (arch == LLMArch::BAILING_MOE) {
+                    config.vocab_size              = 157184;
+                    config.max_position_embeddings = 32768;
+                }
+            } else if (arch == LLMArch::QWEN2) {
+                config.hidden_size             = 1536;
+                config.intermediate_size       = 8960;
+                config.num_heads               = 12;
+                config.num_kv_heads            = 2;
+                config.vocab_size              = 151936;
+                config.max_position_embeddings = 32768;
+                config.bidirectional           = true;
             } else if (arch == LLMArch::GPT_OSS_20B) {
                 config.head_dim                = 64;
                 config.num_heads               = 64;
@@ -283,22 +320,22 @@ namespace LLM {
                     if (contains(name, "attn.q_proj")) {
                         config.llama_cpp_style = true;
                     }
-                    if (contains(name, "visual.patch_embed.proj.1.weight")) {
+                    if (ends_with(name, "visual.patch_embed.proj.1.weight")) {
                         config.vision.split_patch_embed = true;
                     }
-                    if (contains(name, "visual.patch_embed.proj.0.weight")) {
+                    if (ends_with(name, "visual.patch_embed.proj.0.weight")) {
                         config.vision.patch_size  = static_cast<int>(tensor_storage.ne[0]);
                         config.vision.in_channels = tensor_storage.ne[2];
                         config.vision.hidden_size = tensor_storage.ne[3];
                     }
                     // HF-format checkpoints keep the patch embed unsplit under a single name.
-                    if (contains(name, "visual.patch_embed.proj.weight")) {
+                    if (ends_with(name, "visual.patch_embed.proj.weight")) {
                         config.vision.patch_size = static_cast<int>(tensor_storage.ne[0]);
                     }
                     if (contains(name, "visual.patch_embed.bias") || contains(name, "visual.patch_embed.proj.bias")) {
                         config.vision.hidden_size = tensor_storage.ne[0];
                     }
-                    if (contains(name, "visual.pos_embed.weight")) {
+                    if (ends_with(name, "visual.pos_embed.weight") && tensor_storage.n_dims == 2) {
                         config.vision.hidden_size             = tensor_storage.ne[0];
                         config.vision.num_position_embeddings = static_cast<int>(tensor_storage.ne[1]);
                     }
@@ -332,7 +369,7 @@ namespace LLM {
                         }
                     }
                 }
-                if (contains(name, "embed_tokens.weight")) {
+                if (ends_with(name, "embed_tokens.weight") && tensor_storage.n_dims == 2) {
                     config.hidden_size = tensor_storage.ne[0];
                     config.vocab_size  = tensor_storage.ne[1];
                 }
@@ -349,6 +386,41 @@ namespace LLM {
             if ((arch == LLMArch::QWEN3 || arch == LLMArch::QWEN3_VL) && config.num_layers == 28) {
                 config.num_heads = 16;
             }
+            if (arch == LLMArch::QWEN3_5) {
+                config.qwen35_gguf  = tensor_storage_map.count(prefix + ".model.layers.0.linear_attn.A") > 0;
+                config.rms_norm_add = !config.qwen35_gguf;
+                for (const auto& [name, tensor] : tensor_storage_map) {
+                    if (!starts_with(name, prefix + ".")) {
+                        continue;
+                    }
+                    if (ends_with(name, "self_attn.q_norm.weight")) {
+                        config.head_dim = static_cast<int>(tensor.ne[0]);
+                    } else if (ends_with(name, "linear_attn.norm.weight")) {
+                        config.linear_head_dim = static_cast<int>(tensor.ne[0]);
+                    } else if (ends_with(name, "linear_attn.conv1d.weight")) {
+                        config.linear_conv_kernel = static_cast<int>(tensor.ne[0]);
+                        const auto pos            = name.find("layers.");
+                        config.linear_attention_layers.insert(std::stoi(name.substr(pos + 7)));
+                    }
+                }
+                for (const auto& [name, tensor] : tensor_storage_map) {
+                    if (!starts_with(name, prefix + ".")) {
+                        continue;
+                    }
+                    if (ends_with(name, "self_attn.q_proj.weight")) {
+                        config.num_heads = static_cast<int>(tensor.ne[1] / (2 * config.head_dim));
+                    } else if (ends_with(name, "self_attn.k_proj.weight")) {
+                        config.num_kv_heads = static_cast<int>(tensor.ne[1] / config.head_dim);
+                    } else if (ends_with(name, "linear_attn.in_proj_z.weight")) {
+                        config.linear_value_heads = static_cast<int>(tensor.ne[1] / config.linear_head_dim);
+                    }
+                }
+                for (const auto& [name, tensor] : tensor_storage_map) {
+                    if (starts_with(name, prefix + ".") && ends_with(name, "linear_attn.in_proj_qkv.weight")) {
+                        config.linear_key_heads = static_cast<int>((tensor.ne[1] / config.linear_head_dim - config.linear_value_heads) / 2);
+                    }
+                }
+            }
             if (arch == LLMArch::QWEN3_VL &&
                 (config.num_layers == 50 || config.num_layers == 64) &&
                 config.hidden_size == 5120) {
@@ -471,6 +543,8 @@ namespace LLM {
         int64_t n_group;
         int64_t topk_group;
         float routed_scaling_factor;
+        bool image_router;
+        bool fused_experts = false;
 
         void init_params(ggml_context* ctx,
                          const String2TensorStorage& tensor_storage_map = {},
@@ -488,6 +562,10 @@ namespace LLM {
             // scores and the group sums match.
             params["gate.weight"]      = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_experts);
             params["gate.expert_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, num_experts);
+            if (image_router) {
+                params["image_gate.weight"]      = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, num_experts);
+                params["image_gate.expert_bias"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, num_experts);
+            }
 
             ggml_type gate_type = supported_type(get_type(prefix + "experts.gate_proj.weight", tensor_storage_map, GGML_TYPE_F32), hidden_size);
             ggml_type up_type   = supported_type(get_type(prefix + "experts.up_proj.weight", tensor_storage_map, GGML_TYPE_F32), hidden_size);
@@ -506,8 +584,14 @@ namespace LLM {
                 }
             };
 
-            declare_experts("experts.gate_proj.weight", gate_type, hidden_size, moe_intermediate_size);
-            declare_experts("experts.up_proj.weight", up_type, hidden_size, moe_intermediate_size);
+            fused_experts = tensor_storage_map.count(prefix + "experts.gate_up_proj.weight") != 0;
+            if (fused_experts) {
+                auto type = supported_type(get_type(prefix + "experts.gate_up_proj.weight", tensor_storage_map, GGML_TYPE_F32), hidden_size);
+                declare_experts("experts.gate_up_proj.weight", type, hidden_size, 2 * moe_intermediate_size);
+            } else {
+                declare_experts("experts.gate_proj.weight", gate_type, hidden_size, moe_intermediate_size);
+                declare_experts("experts.up_proj.weight", up_type, hidden_size, moe_intermediate_size);
+            }
             declare_experts("experts.down_proj.weight", down_type, moe_intermediate_size, hidden_size);
         }
 
@@ -519,7 +603,8 @@ namespace LLM {
               num_experts_per_tok(config.num_experts_per_tok),
               n_group(config.n_group),
               topk_group(config.topk_group),
-              routed_scaling_factor(config.routed_scaling_factor) {
+              routed_scaling_factor(config.routed_scaling_factor),
+              image_router(config.arch == LLMArch::BAILING_MOE) {
             if (config.num_shared_experts > 0) {
                 blocks["shared_experts"] = std::make_shared<MLP>(config.hidden_size,
                                                                  config.moe_intermediate_size * config.num_shared_experts,
@@ -582,7 +667,7 @@ namespace LLM {
             return ggml_mul_mat_id(ctx->ggml_ctx, w, x, selected_experts);
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* image_mask = nullptr) {
             // x: [N, n_token, hidden_size]
             GGML_ASSERT(num_experts > 0 && num_experts_per_tok > 0);
             GGML_ASSERT(n_group > 0 && topk_group > 0 && num_experts % n_group == 0);
@@ -596,10 +681,21 @@ namespace LLM {
 
             auto logits = ggml_mul_mat(gctx, params["gate.weight"], x);
             logits      = ggml_reshape_2d(gctx, logits, num_experts, n_token_total);
-            auto scores = ggml_sigmoid(gctx, logits);  // [num_experts, tokens]
+            auto bias   = params["gate.expert_bias"];
+            if (image_router) {
+                GGML_ASSERT(image_mask != nullptr);
+                auto mask         = ggml_reshape_2d(gctx, image_mask, 1, n_token_total);
+                auto inverse_mask = ggml_scale_bias(gctx, mask, -1.f, 1.f);
+                auto image_logits = ggml_reshape_2d(gctx, ggml_mul_mat(gctx, params["image_gate.weight"], x), num_experts, n_token_total);
+                logits            = ggml_add(gctx, ggml_mul(gctx, logits, inverse_mask), ggml_mul(gctx, image_logits, mask));
+                bias              = ggml_add(gctx,
+                                             ggml_mul(gctx, ggml_repeat_4d(gctx, bias, num_experts, n_token_total, 1, 1), inverse_mask),
+                                             ggml_mul(gctx, ggml_repeat_4d(gctx, params["image_gate.expert_bias"], num_experts, n_token_total, 1, 1), mask));
+            }
+            auto scores = ggml_sigmoid(gctx, logits);
 
             // The bias steers selection only; the combine weights come from the unbiased scores.
-            auto routing = ggml_add(gctx, scores, params["gate.expert_bias"]);
+            auto routing = ggml_add(gctx, scores, bias);
             routing      = ggml_add(gctx, routing, group_limited_mask(ctx, routing, n_token_total));
 
             auto selected_experts = ggml_argsort_top_k(gctx, routing, (int)num_experts_per_tok);  // [top_k, tokens]
@@ -614,12 +710,17 @@ namespace LLM {
             weights = ggml_scale(gctx, weights, routed_scaling_factor);
             weights = ggml_reshape_3d(gctx, weights, 1, num_experts_per_tok, n_token_total);
 
-            auto xf        = ggml_reshape_3d(gctx, x, hidden_size, 1, n_token_total);
-            auto gate      = expert_linear(ctx, "experts.gate_proj.weight", xf, selected_experts);
-            auto up        = expert_linear(ctx, "experts.up_proj.weight", xf, selected_experts);
-            auto activated = ggml_swiglu_split(gctx, gate, up);
-            auto experts   = expert_linear(ctx, "experts.down_proj.weight", activated, selected_experts);
-            experts        = ggml_mul(gctx, experts, weights);
+            auto xf = ggml_reshape_3d(gctx, x, hidden_size, 1, n_token_total);
+            ggml_tensor* activated;
+            if (fused_experts) {
+                activated = ggml_swiglu(gctx, expert_linear(ctx, "experts.gate_up_proj.weight", xf, selected_experts));
+            } else {
+                auto gate = expert_linear(ctx, "experts.gate_proj.weight", xf, selected_experts);
+                auto up   = expert_linear(ctx, "experts.up_proj.weight", xf, selected_experts);
+                activated = ggml_swiglu_split(gctx, gate, up);
+            }
+            auto experts = expert_linear(ctx, "experts.down_proj.weight", activated, selected_experts);
+            experts      = ggml_mul(gctx, experts, weights);
 
             ggml_tensor* out = nullptr;
             for (int64_t i = 0; i < num_experts_per_tok; ++i) {
@@ -1345,6 +1446,81 @@ namespace LLM {
         }
     };
 
+    struct Qwen35DeltaNet : public UnaryBlock {
+        int head_dim;
+        int key_heads;
+        int value_heads;
+        int conv_dim;
+        int kernel;
+        bool gguf_layout;
+
+        Qwen35DeltaNet(const LLMConfig& config)
+            : head_dim(config.linear_head_dim), key_heads(config.linear_key_heads), value_heads(config.linear_value_heads), conv_dim(head_dim * (key_heads * 2 + value_heads)), kernel(config.linear_conv_kernel), gguf_layout(config.qwen35_gguf) {
+            auto hidden_size      = config.hidden_size;
+            blocks["in_proj_qkv"] = std::make_shared<Linear>(hidden_size, conv_dim, false);
+            blocks["in_proj_z"]   = std::make_shared<Linear>(hidden_size, head_dim * value_heads, false);
+            blocks["in_proj_a"]   = std::make_shared<Linear>(hidden_size, value_heads, false);
+            blocks["in_proj_b"]   = std::make_shared<Linear>(hidden_size, value_heads, false);
+            blocks["out_proj"]    = std::make_shared<Linear>(head_dim * value_heads, hidden_size, false);
+            blocks["norm"]        = std::make_shared<LLMRMSNorm>(head_dim, 1e-6f);
+        }
+
+        void init_params(ggml_context* ctx, const String2TensorStorage& = {}, std::string = "") override {
+            params["conv1d.weight"]             = gguf_layout ? ggml_new_tensor_2d(ctx, GGML_TYPE_F32, kernel, conv_dim) : ggml_new_tensor_3d(ctx, GGML_TYPE_F32, kernel, 1, conv_dim);
+            params[gguf_layout ? "A" : "A_log"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, value_heads);
+            params["dt_bias"]                   = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, value_heads);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
+            auto c          = ctx->ggml_ctx;
+            const int64_t n = x->ne[1];
+            GGML_ASSERT(x->ne[2] == 1);
+            auto linear        = [&](const char* name) { return std::dynamic_pointer_cast<Linear>(blocks[name])->forward(ctx, x); };
+            auto qkv           = linear("in_proj_qkv");
+            auto padded        = ggml_pad_ext(c, qkv, 0, 0, kernel - 1, 0, 0, 0, 0, 0);
+            ggml_tensor* mixed = nullptr;
+            // Causal depthwise convolution, with zero state for every independent prompt.
+            for (int i = 0; i < kernel; ++i) {
+                auto w    = ggml_cont(c, ggml_view_2d(c, params["conv1d.weight"], 1, conv_dim, kernel * sizeof(float), i * sizeof(float)));
+                w         = ggml_reshape_1d(c, w, conv_dim);
+                auto part = ggml_mul(c, ggml_ext_slice(c, padded, 1, i, i + n), w);
+                mixed     = mixed ? ggml_add(c, mixed, part) : part;
+            }
+            mixed      = ggml_silu(c, mixed);
+            auto heads = [&](int start, int count) {
+                auto h = ggml_ext_slice(c, mixed, 0, start, start + head_dim * count);
+                return ggml_reshape_4d(c, h, head_dim, count, n, 1);
+            };
+            auto repeat_keys = [&](ggml_tensor* h) {
+                h = ggml_scale(c, ggml_rms_norm(c, h, 1e-6f / head_dim), 1.f / std::sqrt(float(head_dim)));
+                // llama.cpp reorders V heads and stores negative exp(A_log) and offset norms.
+                if (gguf_layout) {
+                    return h;
+                }
+                // ggml GDN broadcasts heads cyclically; Qwen uses adjacent repeated heads.
+                h          = ggml_reshape_4d(c, h, head_dim, 1, key_heads, n);
+                auto shape = ggml_new_tensor_4d(c, GGML_TYPE_F32, head_dim, value_heads / key_heads, key_heads, n);
+                return ggml_reshape_4d(c, ggml_repeat(c, h, shape), head_dim, value_heads, n, 1);
+            };
+            auto q     = repeat_keys(heads(0, key_heads));
+            auto k     = repeat_keys(heads(head_dim * key_heads, key_heads));
+            auto v     = heads(2 * head_dim * key_heads, value_heads);
+            auto beta  = ggml_reshape_4d(c, ggml_sigmoid(c, linear("in_proj_b")), 1, value_heads, n, 1);
+            auto decay = ggml_softplus(c, ggml_add(c, linear("in_proj_a"), params["dt_bias"]));
+            auto a     = gguf_layout ? params["A"] : ggml_neg(c, ggml_exp(c, params["A_log"]));
+            decay      = ggml_mul(c, decay, a);
+            decay      = ggml_reshape_4d(c, decay, 1, value_heads, n, 1);
+            auto state = ggml_ext_zeros(c, head_dim, head_dim, value_heads, 1);
+            auto out   = ggml_gated_delta_net(c, q, k, v, decay, beta, state, 1);
+            out        = ggml_view_3d(c, out, head_dim, value_heads, n, head_dim * sizeof(float), head_dim * value_heads * sizeof(float), 0);
+            out        = std::dynamic_pointer_cast<LLMRMSNorm>(blocks["norm"])->forward(ctx, out);
+            auto z     = ggml_reshape_3d(c, linear("in_proj_z"), head_dim, value_heads, n);
+            out        = ggml_mul(c, out, ggml_silu(c, z));
+            out        = ggml_reshape_3d(c, out, head_dim * value_heads, n, 1);
+            return std::dynamic_pointer_cast<Linear>(blocks["out_proj"])->forward(ctx, out);
+        }
+    };
+
     struct Attention : public GGMLBlock {
     protected:
         LLMArch arch;
@@ -1394,7 +1570,7 @@ namespace LLM {
                 GGML_ASSERT(!k_eq_v);
                 blocks["query_key_value"] = std::make_shared<Linear>(config.hidden_size, (num_heads + num_kv_heads * 2) * head_dim, config.qkv_bias);
             } else {
-                blocks["q_proj"] = std::make_shared<Linear>(config.hidden_size, num_heads * head_dim, config.qkv_bias);
+                blocks["q_proj"] = std::make_shared<Linear>(config.hidden_size, num_heads * head_dim * (arch == LLMArch::QWEN3_5 ? 2 : 1), config.qkv_bias);
                 blocks["k_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
                 if (!k_eq_v) {
                     blocks["v_proj"] = std::make_shared<Linear>(config.hidden_size, num_kv_heads * head_dim, config.qkv_bias);
@@ -1428,7 +1604,9 @@ namespace LLM {
                              ggml_tensor* x,
                              ggml_tensor* input_pos,
                              ggml_tensor* attention_mask = nullptr,
-                             int rope_index              = 0) {
+                             int rope_index              = 0,
+                             ggml_tensor* rope_cos       = nullptr,
+                             ggml_tensor* rope_sin       = nullptr) {
             // x: [N, n_token, hidden_size]
             int64_t n_token = x->ne[1];
             int64_t N       = x->ne[2];
@@ -1455,6 +1633,13 @@ namespace LLM {
                 v = k_eq_v ? k : v_proj->forward(ctx, x);  // [N, n_token, num_kv_heads*head_dim]
             }
 
+            ggml_tensor* output_gate = nullptr;
+            if (arch == LLMArch::QWEN3_5) {
+                q           = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim * 2, num_heads, n_token, N);
+                output_gate = ggml_ext_slice(ctx->ggml_ctx, q, 0, head_dim, head_dim * 2);
+                output_gate = ggml_reshape_3d(ctx->ggml_ctx, output_gate, head_dim * num_heads, n_token, N);
+                q           = ggml_ext_slice(ctx->ggml_ctx, q, 0, 0, head_dim);
+            }
             q = ggml_reshape_4d(ctx->ggml_ctx, q, head_dim, num_heads, n_token, N);     // [N, n_token, num_heads, head_dim]
             k = ggml_reshape_4d(ctx->ggml_ctx, k, head_dim, num_kv_heads, n_token, N);  // [N, n_token, num_kv_heads, head_dim]
             v = ggml_reshape_4d(ctx->ggml_ctx, v, head_dim, num_kv_heads, n_token, N);  // [N, n_token, num_kv_heads, head_dim]
@@ -1471,13 +1656,28 @@ namespace LLM {
                 v = ggml_rms_norm(ctx->ggml_ctx, v, rms_norm_eps);
             }
 
-            if (arch == LLMArch::MISTRAL_SMALL_3_2) {
+            if (rope_cos != nullptr) {
+                GGML_ASSERT(rope_sin != nullptr);
+                // Bailing video RoPE interleaves spatial frequencies within a partial NEOX head.
+                auto rotate = [&](ggml_tensor* input) {
+                    auto gctx    = ctx->ggml_ctx;
+                    int64_t half = rope_cos->ne[0];
+                    auto first   = ggml_ext_slice(gctx, input, 0, 0, half);
+                    auto second  = ggml_ext_slice(gctx, input, 0, half, 2 * half);
+                    auto left    = ggml_sub(gctx, ggml_mul(gctx, first, rope_cos), ggml_mul(gctx, second, rope_sin));
+                    auto right   = ggml_add(gctx, ggml_mul(gctx, second, rope_cos), ggml_mul(gctx, first, rope_sin));
+                    auto rotated = ggml_concat(gctx, left, right, 0);
+                    return ggml_concat(gctx, rotated, ggml_ext_slice(gctx, input, 0, 2 * half, input->ne[0]), 0);
+                };
+                q = rotate(q);
+                k = rotate(k);
+            } else if (arch == LLMArch::MISTRAL_SMALL_3_2) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NORMAL, 8192, 1000000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
                 k = ggml_rope_ext(ctx->ggml_ctx, k, input_pos, nullptr, 128, GGML_ROPE_TYPE_NORMAL, 8192, 1000000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
             } else if (arch == LLMArch::MINISTRAL_3_3B) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NEOX, 262144, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
                 k = ggml_rope_ext(ctx->ggml_ctx, k, input_pos, nullptr, 128, GGML_ROPE_TYPE_NEOX, 262144, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
-            } else if (arch == LLMArch::QWEN3) {
+            } else if (arch == LLMArch::QWEN3 || arch == LLMArch::QWEN2) {
                 q = ggml_rope_ext(ctx->ggml_ctx, q, input_pos, nullptr, 128, GGML_ROPE_TYPE_NEOX, 40960, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
                 k = ggml_rope_ext(ctx->ggml_ctx, k, input_pos, nullptr, 128, GGML_ROPE_TYPE_NEOX, 40960, 1000000.f, 1.f, 0.f, 1.f, 32.f, 1.f);
             } else if (arch == LLMArch::GPT_OSS_20B) {
@@ -1596,7 +1796,7 @@ namespace LLM {
                                   1.f,
                                   32.f,
                                   1.f);
-            } else if (arch == LLMArch::LLADA2_MOE) {
+            } else if (arch == LLMArch::LLADA2_MOE || arch == LLMArch::QWEN3_5) {
                 // LLaDA2 slices the head (query[..., :rotary_dim]) instead of zero-padding
                 // inv_freq like gemma does, so rotate_half pairs i with i + rotary_dim/2 and the
                 // frequencies use rotary_dim as the exponent denominator. Passing n_dims =
@@ -1655,7 +1855,7 @@ namespace LLM {
                 v_attn      = ggml_reshape_3d(ctx->ggml_ctx, v_attn, n_token, head_dim, num_kv_heads * N);
 
                 auto kq = ggml_mul_mat(ctx->ggml_ctx, k, q);
-                ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
+                ggml_prec_set_acc(kq, GGML_PREC_F32);
                 kq = ggml_scale_inplace(ctx->ggml_ctx, kq, 1.0f / std::sqrt(static_cast<float>(head_dim)));
                 if (attention_mask != nullptr) {
                     kq = ggml_add_inplace(ctx->ggml_ctx, kq, attention_mask);
@@ -1672,6 +1872,9 @@ namespace LLM {
                 x = ggml_ext_attention_ext(ctx, q, k, v, num_heads, attention_mask, true, ctx->flash_attn_enabled);  // [N, n_token, hidden_size]
             }
 
+            if (output_gate) {
+                x = ggml_mul(ctx->ggml_ctx, x, ggml_sigmoid(ctx->ggml_ctx, output_gate));
+            }
             x = out_proj->forward(ctx, x);  // [N, n_token, hidden_size]
             return x;
         }
@@ -1721,10 +1924,14 @@ namespace LLM {
             if (!config.sliding_attention.empty()) {
                 sliding_attention = config.sliding_attention[layer_index % config.sliding_attention.size()];
             }
-            blocks["self_attn"] = std::make_shared<Attention>(config, sliding_attention == 0);
+            if (config.arch == LLMArch::QWEN3_5 && config.linear_attention_layers.count(layer_index)) {
+                blocks["linear_attn"] = std::make_shared<Qwen35DeltaNet>(config);
+            } else {
+                blocks["self_attn"] = std::make_shared<Attention>(config, sliding_attention == 0);
+            }
             if (config.arch == LLMArch::GPT_OSS_20B) {
                 blocks["mlp"] = std::make_shared<GPTOSSMLP>(config);
-            } else if (config.arch == LLMArch::LLADA2_MOE && layer_index >= config.first_k_dense_replace) {
+            } else if ((config.arch == LLMArch::LLADA2_MOE || config.arch == LLMArch::BAILING_MOE) && layer_index >= config.first_k_dense_replace) {
                 blocks["mlp"] = std::make_shared<LLaDA2MoEMLP>(config);
             } else {
                 blocks["mlp"] = std::make_shared<MLP>(config.hidden_size,
@@ -1746,9 +1953,12 @@ namespace LLM {
                              ggml_tensor* x,
                              ggml_tensor* input_pos,
                              ggml_tensor* attention_mask         = nullptr,
-                             ggml_tensor* sliding_attention_mask = nullptr) {
+                             ggml_tensor* sliding_attention_mask = nullptr,
+                             ggml_tensor* image_mask             = nullptr,
+                             ggml_tensor* rope_cos               = nullptr,
+                             ggml_tensor* rope_sin               = nullptr) {
             // x: [N, n_token, hidden_size]
-            auto self_attn                                  = std::dynamic_pointer_cast<Attention>(blocks["self_attn"]);
+            auto self_attn                                  = blocks.count("self_attn") ? std::dynamic_pointer_cast<Attention>(blocks["self_attn"]) : nullptr;
             auto input_layernorm                            = std::dynamic_pointer_cast<LLMRMSNorm>(blocks["input_layernorm"]);
             auto pre_ffw_norm                               = std::dynamic_pointer_cast<LLMRMSNorm>(blocks[pre_ffw_norm_name]);
             std::shared_ptr<LLMRMSNorm> post_attention_norm = nullptr;
@@ -1768,7 +1978,11 @@ namespace LLM {
 
             auto residual = x;
             x             = input_layernorm->forward(ctx, x);
-            x             = self_attn->forward(ctx, x, input_pos, block_attention_mask, rope_index);
+            if (blocks.count("linear_attn")) {
+                x = std::dynamic_pointer_cast<Qwen35DeltaNet>(blocks["linear_attn"])->forward(ctx, x);
+            } else {
+                x = self_attn->forward(ctx, x, input_pos, block_attention_mask, rope_index, rope_cos, rope_sin);
+            }
             if (post_attention_norm != nullptr) {
                 x = post_attention_norm->forward(ctx, x);
             }
@@ -1782,7 +1996,7 @@ namespace LLM {
             } else if (auto moe_mlp = std::dynamic_pointer_cast<LLaDA2MoEMLP>(blocks["mlp"])) {
                 // LLaDA2 is dense for the first first_k_dense_replace layers and MoE afterwards,
                 // so the block type varies per layer rather than per arch.
-                x = moe_mlp->forward(ctx, x);
+                x = moe_mlp->forward(ctx, x, image_mask);
             } else {
                 auto mlp = std::dynamic_pointer_cast<MLP>(blocks["mlp"]);
                 x        = mlp->forward(ctx, x);
@@ -1804,10 +2018,11 @@ namespace LLM {
     protected:
         int64_t num_layers;
         LLMConfig config;
+        std::string graph_cut_prefix;
 
     public:
-        TextModel(const LLMConfig& config)
-            : num_layers(config.num_layers), config(config) {
+        TextModel(const LLMConfig& config, const std::string& graph_cut_prefix = "llm.text")
+            : num_layers(config.num_layers), config(config), graph_cut_prefix(graph_cut_prefix) {
             blocks["embed_tokens"] = std::shared_ptr<GGMLBlock>(new Embedding(config.vocab_size, config.hidden_size));
             for (int i = 0; i < num_layers; i++) {
                 blocks["layers." + std::to_string(i)] = std::shared_ptr<GGMLBlock>(new TransformerBlock(config, i));
@@ -1831,7 +2046,10 @@ namespace LLM {
                                     std::set<int> out_layers,
                                     const std::vector<std::vector<std::pair<int, ggml_tensor*>>>& deepstack_image_embeds = {},
                                     ggml_tensor* sliding_attention_mask                                                  = nullptr,
-                                    bool return_all_hidden_states                                                        = false) {
+                                    bool return_all_hidden_states                                                        = false,
+                                    ggml_tensor* image_mask                                                              = nullptr,
+                                    ggml_tensor* rope_cos                                                                = nullptr,
+                                    ggml_tensor* rope_sin                                                                = nullptr) {
             auto norm = config.final_norm ? std::dynamic_pointer_cast<LLMRMSNorm>(blocks["norm"])
                                           : nullptr;
             std::vector<ggml_tensor*> intermediate_outputs;
@@ -1843,18 +2061,18 @@ namespace LLM {
                 intermediate_outputs.push_back(x);
             }
 
-            sd::ggml_graph_cut::mark_graph_cut(x, "llm.text.prelude", "x");
+            sd::ggml_graph_cut::mark_graph_cut(x, graph_cut_prefix + ".prelude", "x");
             for (int i = 0; i < num_layers; i++) {
                 auto block = std::dynamic_pointer_cast<TransformerBlock>(blocks["layers." + std::to_string(i)]);
 
-                x = block->forward(ctx, x, input_pos, attention_mask, sliding_attention_mask);
+                x = block->forward(ctx, x, input_pos, attention_mask, sliding_attention_mask, image_mask, rope_cos, rope_sin);
                 if (i < static_cast<int>(deepstack_image_embeds.size())) {
                     x = add_deepstack_image_embeds(ctx, x, deepstack_image_embeds[static_cast<size_t>(i)]);
                 }
                 if (return_all_hidden_states || out_layers.size() > 1) {
                     x = ggml_cont(ctx->ggml_ctx, x);
                 }
-                sd::ggml_graph_cut::mark_graph_cut(x, "llm.text.layers." + std::to_string(i), "x");
+                sd::ggml_graph_cut::mark_graph_cut(x, graph_cut_prefix + ".layers." + std::to_string(i), "x");
                 if (return_all_hidden_states) {
                     if (i + 1 < num_layers) {
                         intermediate_outputs.push_back(x);
@@ -2269,7 +2487,8 @@ namespace LLM {
                                  const DeepStackImageEmbeds& deepstack_image_embeds_tensor,
                                  const std::vector<ImageGrid>& image_grids,
                                  std::set<int> out_layers,
-                                 bool return_all_hidden_states = false) {
+                                 bool return_all_hidden_states           = false,
+                                 const sd::Tensor<int32_t>& position_ids = {}) {
             ggml_cgraph* gf        = new_graph_custom(LLM_GRAPH_SIZE);
             ggml_tensor* input_ids = make_input(input_ids_tensor);
             std::vector<std::pair<int, ggml_tensor*>> image_embeds;
@@ -2290,6 +2509,7 @@ namespace LLM {
             if (config.arch == LLMArch::MISTRAL_SMALL_3_2 ||
                 config.arch == LLMArch::MINISTRAL_3_3B ||
                 config.arch == LLMArch::QWEN3 ||
+                config.arch == LLMArch::QWEN3_5 ||
                 config.arch == LLMArch::GEMMA3_12B ||
                 config.arch == LLMArch::GEMMA4_12B ||
                 config.arch == LLMArch::GEMMA2_2B ||
@@ -2333,6 +2553,10 @@ namespace LLM {
                 }
             }
 
+            if (!position_ids.empty()) {
+                GGML_ASSERT(position_ids.numel() == static_cast<int64_t>(input_pos_vec.size()));
+                input_pos_vec = position_ids.values();
+            }
             auto input_pos = ggml_new_tensor_1d(compute_ctx,
                                                 GGML_TYPE_I32,
                                                 input_pos_vec.size());
@@ -2406,7 +2630,8 @@ namespace LLM {
                                   bool return_all_hidden_states                      = false,
                                   bool auto_runner_end                               = true,
                                   const DeepStackImageEmbeds& deepstack_image_embeds = {},
-                                  const std::vector<ImageGrid>& image_grids          = {}) {
+                                  const std::vector<ImageGrid>& image_grids          = {},
+                                  const sd::Tensor<int32_t>& position_ids            = {}) {
             auto get_graph = [&]() -> ggml_cgraph* {
                 return build_graph(input_ids,
                                    attention_mask,
@@ -2414,7 +2639,8 @@ namespace LLM {
                                    deepstack_image_embeds,
                                    image_grids,
                                    out_layers,
-                                   return_all_hidden_states);
+                                   return_all_hidden_states,
+                                   position_ids);
             };
             return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, auto_runner_end),
                                                    input_ids.dim() + 1);
diff --git a/src/model/te/ming_image_te.hpp b/src/model/te/ming_image_te.hpp
new file mode 100644
index 0000000000..ab3b8e6dbc
--- /dev/null
+++ b/src/model/te/ming_image_te.hpp
@@ -0,0 +1,159 @@
+#ifndef __SD_MODEL_TE_MING_IMAGE_TE_HPP__
+#define __SD_MODEL_TE_MING_IMAGE_TE_HPP__
+
+#include "llm.hpp"
+
+namespace MingImageTE {
+    struct MingImageTEConfig {
+        LLM::LLMConfig backbone;
+        LLM::LLMConfig connector;
+        int64_t num_queries   = 256;
+        int64_t caption_dim   = 2560;
+        int64_t diffusion_dim = 3840;
+
+        static MingImageTEConfig detect_from_weights(const String2TensorStorage& tensors, const std::string& prefix) {
+            MingImageTEConfig config;
+            for (const auto& entry : tensors) {
+                if (starts_with(entry.first, prefix + ".backbone.") &&
+                    contains(entry.first, ".mlp.experts.") && entry.second.type == GGML_TYPE_I8) {
+                    throw std::runtime_error("Ming-Image INT8/W4A8 text encoder experts are not supported; use the BF16 text encoder");
+                }
+            }
+            bool vision           = false;
+            config.backbone       = LLM::LLMConfig::detect_from_weights(tensors, prefix + ".backbone.", LLM::LLMArch::BAILING_MOE, vision);
+            config.connector      = LLM::LLMConfig::detect_from_weights(tensors, prefix + ".connector.", LLM::LLMArch::QWEN2, vision);
+            const auto query      = tensors.find(prefix + ".query_tokens_dict.16x16");
+            const auto projection = tensors.find(prefix + ".proj_out.weight");
+            const auto direct     = tensors.find(prefix + ".proj_directvlm.1.weight");
+            if (query == tensors.end() || projection == tensors.end() || direct == tensors.end()) {
+                throw std::runtime_error("Ming-Image requires the learned queries, connector and both condition projections");
+            }
+            config.num_queries   = query->second.ne[1];
+            config.caption_dim   = projection->second.ne[1];
+            config.diffusion_dim = direct->second.ne[1];
+            if (config.num_queries != 256 || config.caption_dim != 2560 || config.diffusion_dim != 3840 ||
+                config.backbone.num_layers != 20 || config.backbone.hidden_size != 2048 ||
+                config.connector.num_layers != 28 || config.connector.hidden_size != 1536) {
+                throw std::runtime_error("unsupported Ming-Image text encoder configuration");
+            }
+            LOG_VERBOSE("ming_image_te: queries = %" PRId64 ", caption_dim = %" PRId64 ", diffusion_dim = %" PRId64,
+                        config.num_queries, config.caption_dim, config.diffusion_dim);
+            return config;
+        }
+    };
+
+    struct ConnectorModel : LLM::TextModel {
+        explicit ConnectorModel(const LLM::LLMConfig& config)
+            : LLM::TextModel(config, "ming_image.connector") {
+            blocks.erase("embed_tokens");
+        }
+    };
+
+    class MingImageTextModel : public GGMLBlock {
+        MingImageTEConfig config;
+
+        void init_params(ggml_context* ctx, const String2TensorStorage& tensors = {}, const std::string prefix = "") override {
+            params["query_tokens_dict.16x16"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, config.backbone.hidden_size, config.num_queries);
+        }
+
+    public:
+        explicit MingImageTextModel(const MingImageTEConfig& config)
+            : config(config) {
+            blocks["backbone"]         = std::make_shared<LLM::TextModel>(config.backbone, "ming_image.backbone");
+            blocks["connector"]        = std::make_shared<ConnectorModel>(config.connector);
+            blocks["proj_in"]          = std::make_shared<Linear>(config.backbone.hidden_size, config.connector.hidden_size);
+            blocks["proj_out"]         = std::make_shared<Linear>(config.connector.hidden_size, config.caption_dim);
+            blocks["proj_directvlm.0"] = std::make_shared<RMSNorm>(config.backbone.hidden_size * 3, 1e-5f);
+            blocks["proj_directvlm.1"] = std::make_shared<Linear>(config.backbone.hidden_size * 3, config.diffusion_dim);
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* ids, int64_t prompt_length, ggml_tensor* mask, ggml_tensor* image_mask, ggml_tensor* cos, ggml_tensor* sin, ggml_tensor* connector_positions) {
+            auto gctx                 = ctx->ggml_ctx;
+            auto backbone             = std::dynamic_pointer_cast<LLM::TextModel>(blocks["backbone"]);
+            auto connector            = std::dynamic_pointer_cast<ConnectorModel>(blocks["connector"]);
+            auto x                    = backbone->embed(ctx, ids);
+            const int64_t query_start = prompt_length + 1;
+            auto before               = ggml_ext_slice(gctx, x, 1, 0, query_start);
+            auto after                = ggml_ext_slice(gctx, x, 1, query_start + config.num_queries, x->ne[1]);
+            x                         = ggml_concat(gctx, ggml_concat(gctx, before, params["query_tokens_dict.16x16"], 1), after, 1);
+            // HF hidden_states[20] includes final RMSNorm; sd.cpp selects it as num_layers + 1.
+            x            = backbone->forward_embeds(ctx, x, nullptr, mask, {5, 12, 21}, {}, nullptr, false, image_mask, cos, sin);
+            auto direct  = ggml_ext_slice(gctx, x, 1, 0, prompt_length);
+            direct       = std::dynamic_pointer_cast<RMSNorm>(blocks["proj_directvlm.0"])->forward(ctx, direct);
+            direct       = std::dynamic_pointer_cast<Linear>(blocks["proj_directvlm.1"])->forward(ctx, direct);
+            auto queries = ggml_ext_slice(gctx, x, 0, config.backbone.hidden_size * 2, config.backbone.hidden_size * 3);
+            queries      = ggml_ext_slice(gctx, queries, 1, query_start, query_start + config.num_queries);
+            queries      = std::dynamic_pointer_cast<Linear>(blocks["proj_in"])->forward(ctx, queries);
+            queries      = connector->forward_embeds(ctx, queries, connector_positions, nullptr, {});
+            queries      = std::dynamic_pointer_cast<Linear>(blocks["proj_out"])->forward(ctx, queries);
+            queries      = ggml_pad(gctx, queries, static_cast<int>(config.diffusion_dim - config.caption_dim), 0, 0, 0);
+            return ggml_concat(gctx, queries, direct, 1);
+        }
+    };
+
+    struct MingImageTextRunner : GGMLRunner {
+        MingImageTEConfig config;
+        MingImageTextModel model;
+
+        MingImageTextRunner(ggml_backend_t backend, const String2TensorStorage& tensors, const std::string& prefix, std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+            : GGMLRunner(backend, weight_manager), config(MingImageTEConfig::detect_from_weights(tensors, prefix)), model(config) {
+            model.init(params_ctx, tensors, prefix);
+        }
+
+        std::string get_desc() override { return "ming_image_text"; }
+
+        void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors, const std::string& prefix) {
+            model.get_param_tensors(tensors, prefix);
+        }
+
+        void get_param_tensor_ops(std::map<ggml_tensor*, enum ggml_op>& ops) {
+            model.get_param_tensor_ops(ops);
+        }
+
+        sd::Tensor<float> compute(int n_threads, const std::vector<int>& tokens) {
+            const int64_t prompt_length = tokens.size();
+            const int64_t query_start   = prompt_length + 1;
+            const int64_t total         = prompt_length + config.num_queries + 2;
+            std::vector<int32_t> ids(tokens.begin(), tokens.end());
+            ids.push_back(157158);
+            ids.insert(ids.end(), config.num_queries, 157157);
+            ids.push_back(157159);
+            auto input = sd::Tensor<int32_t>({total}, std::move(ids));
+            sd::Tensor<float> attention_mask({total, total});
+            sd::Tensor<float> image_mask({1, total});
+            sd::Tensor<float> cos({32, 1, total}), sin({32, 1, total});
+            std::vector<int32_t> positions(config.num_queries);
+            std::iota(positions.begin(), positions.end(), 0);
+            auto connector_positions = sd::Tensor<int32_t>({config.num_queries}, std::move(positions));
+            for (int64_t token = 0; token < total; ++token) {
+                const bool query  = token >= query_start && token < query_start + config.num_queries;
+                image_mask[token] = query ? 1.f : 0.f;
+                for (int64_t key = 0; key < total; ++key) {
+                    attention_mask[key + token * total] = key > token ? -INFINITY : 0.f;
+                }
+                // A 16x16 query bank is represented upstream as a [1, 2, 512] image grid,
+                // then spatially merged and centered to [1, 1, 256].
+                int64_t temporal = query ? query_start : (token == total - 1 ? query_start + 1 : token);
+                int64_t width    = query ? token - 127 : temporal;
+                for (int j = 0; j < 32; ++j) {
+                    int64_t position    = query && j < 24 && j % 2 ? width : temporal;
+                    float frequency     = 1.f / std::pow(600000.f, static_cast<float>(2 * j) / 64.f);
+                    float angle         = static_cast<float>(position) * frequency;
+                    cos[token * 32 + j] = std::cos(angle);
+                    sin[token * 32 + j] = std::sin(angle);
+                }
+            }
+            auto graph = [&]() {
+                auto gf  = new_graph_custom(LLM::LLM_GRAPH_SIZE);
+                auto ctx = get_context();
+                auto out = model.forward(&ctx, make_input(input), prompt_length, make_input(attention_mask),
+                                         make_input(image_mask), make_input(cos), make_input(sin), make_input(connector_positions));
+                ggml_build_forward_expand(gf, out);
+                return gf;
+            };
+            return take_or_empty(GGMLRunner::compute(graph, n_threads));
+        }
+    };
+}
+
+#endif  // __SD_MODEL_TE_MING_IMAGE_TE_HPP__
diff --git a/src/model/upscaler/mosr.hpp b/src/model/upscaler/mosr.hpp
new file mode 100644
index 0000000000..0aa158c81c
--- /dev/null
+++ b/src/model/upscaler/mosr.hpp
@@ -0,0 +1,337 @@
+#ifndef __SD_MODEL_UPSCALER_MOSR_HPP__
+#define __SD_MODEL_UPSCALER_MOSR_HPP__
+
+#include <algorithm>
+#include <cmath>
+#include <map>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "core/ggml_extend.h"
+#include "core/ggml_runner.h"
+#include "core/util.h"
+#include "model/common/ggml_block.hpp"
+
+/*
+    ===================================    MoSR  ===================================
+    References:
+    https://github.com/umzi2/MoSR
+    https://github.com/chaiNNer-org/spandrel/blob/main/libs/spandrel/spandrel/architectures/MoSR/__arch/mosr_arch.py
+
+    Only the PixelShuffle ("ps") upsampler is supported.
+*/
+
+struct MoSRConfig {
+    int scale       = 4;
+    int n_block     = 24;
+    int in_ch       = 3;
+    int dim         = 64;
+    int hidden      = 96;
+    int conv_ch     = 64;
+    int kernel_size = 7;
+
+    bool is_valid() const {
+        // The upscaler feeds RGB tiles, so only 3-channel models are usable.
+        return in_ch == 3 && scale > 0 && n_block > 0 && conv_ch <= hidden;
+    }
+
+    static MoSRConfig detect_from_weights(const String2TensorStorage& tensor_storage_map,
+                                          const std::string& prefix = "") {
+        MoSRConfig config;
+        auto find_weight = [&](const std::string& suffix) -> const TensorStorage* {
+            std::string name = prefix.empty() ? suffix : prefix + "." + suffix;
+            auto iter        = tensor_storage_map.find(name);
+            if (iter == tensor_storage_map.end() || iter->second.n_dims != 4) {
+                return nullptr;
+            }
+            return &iter->second;
+        };
+
+        const TensorStorage* first = find_weight("gblocks.0.weight");
+        const TensorStorage* fc1   = find_weight("gblocks.1.fc1.weight");
+        const TensorStorage* conv  = find_weight("gblocks.1.conv.weight");
+        const TensorStorage* up    = find_weight("upsampler.0.weight");
+        if (first == nullptr || fc1 == nullptr || conv == nullptr || up == nullptr) {
+            // Missing tensors, or the unsupported "gps"/"dys" upsamplers.
+            config.n_block = 0;
+            return config;
+        }
+
+        int seq_len                      = 0;
+        const std::string gblocks_prefix = prefix.empty() ? "gblocks." : prefix + ".gblocks.";
+        for (const auto& [name, _] : tensor_storage_map) {
+            if (!starts_with(name, gblocks_prefix)) {
+                continue;
+            }
+            size_t pos = name.find('.', gblocks_prefix.size());
+            if (pos == std::string::npos) {
+                continue;
+            }
+            try {
+                int idx = std::stoi(name.substr(gblocks_prefix.size(), pos - gblocks_prefix.size()));
+                seq_len = std::max(seq_len, idx + 1);
+            } catch (...) {
+            }
+        }
+
+        // gblocks = [conv_in, n_block x GatedCNNBlock, conv, Mish, conv, Mish, conv_1x1]
+        config.n_block     = seq_len - 6;
+        config.in_ch       = (int)first->ne[2];
+        config.dim         = (int)first->ne[3];
+        config.hidden      = (int)fc1->ne[3] / 2;
+        config.conv_ch     = (int)conv->ne[3];
+        config.kernel_size = (int)conv->ne[0];
+        config.scale       = (int)std::lround(std::sqrt((double)up->ne[3] / config.in_ch));
+        if (config.scale * config.scale * config.in_ch != up->ne[3]) {
+            config.scale = 0;
+        }
+
+        LOG_VERBOSE("mosr: scale = %d, n_block = %d, in_ch = %d, dim = %d, hidden = %d, conv_ch = %d, kernel_size = %d",
+                    config.scale,
+                    config.n_block,
+                    config.in_ch,
+                    config.dim,
+                    config.hidden,
+                    config.conv_ch,
+                    config.kernel_size);
+        return config;
+    }
+};
+
+static inline ggml_tensor* mosr_mish(ggml_context* ctx, ggml_tensor* x) {
+    return ggml_mul(ctx, x, ggml_tanh(ctx, ggml_softplus(ctx, x)));
+}
+
+class MoSRLayerNorm2d : public UnaryBlock {
+protected:
+    int64_t channels;
+
+    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
+        params["weight"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, channels);
+        params["bias"]   = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, channels);
+    }
+
+public:
+    explicit MoSRLayerNorm2d(int64_t channels)
+        : channels(channels) {}
+
+    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
+        // [N, C, H, W] -> [N, H, W, C] so the norm reduces over channels.
+        x = ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 2, 0, 3));
+        x = ggml_ext_layer_norm(ctx->ggml_ctx, x, params["weight"], params["bias"], 1e-6f);
+        return ggml_ext_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 2, 0, 1, 3));
+    }
+};
+
+class MoSRDepthwiseConv2d : public UnaryBlock {
+protected:
+    int64_t channels;
+    int kernel_size;
+
+    void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
+        // The direct depthwise kernels of the CUDA and Vulkan backends need an F32 kernel.
+        params["weight"] = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, kernel_size, kernel_size, 1, channels);
+        params["bias"]   = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, channels);
+    }
+
+public:
+    MoSRDepthwiseConv2d(int64_t channels, int kernel_size)
+        : channels(channels), kernel_size(kernel_size) {}
+
+    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override {
+        const int pad = kernel_size / 2;
+        x             = ggml_conv_2d_dw_direct(ctx->ggml_ctx, params["weight"], ggml_ext_cont(ctx->ggml_ctx, x), 1, 1, pad, pad, 1, 1);
+        auto b        = ggml_reshape_4d(ctx->ggml_ctx, params["bias"], 1, 1, channels, 1);
+        return ggml_add_inplace(ctx->ggml_ctx, x, b);
+    }
+};
+
+class MoSRGatedCNNBlock : public GGMLBlock {
+protected:
+    int hidden;
+    int conv_ch;
+
+public:
+    MoSRGatedCNNBlock(int dim, int hidden, int conv_ch, int kernel_size)
+        : hidden(hidden), conv_ch(conv_ch) {
+        blocks["norm"] = std::shared_ptr<GGMLBlock>(new MoSRLayerNorm2d(dim));
+        blocks["fc1"]  = std::shared_ptr<GGMLBlock>(new Conv2d(dim, hidden * 2, {3, 3}, {1, 1}, {1, 1}));
+        blocks["conv"] = std::shared_ptr<GGMLBlock>(new MoSRDepthwiseConv2d(conv_ch, kernel_size));
+        blocks["fc2"]  = std::shared_ptr<GGMLBlock>(new Conv2d(hidden, dim, {3, 3}, {1, 1}, {1, 1}));
+    }
+
+    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+        // x: [n, dim, h, w]
+        // return: [n, dim, h, w]
+        auto norm = std::dynamic_pointer_cast<MoSRLayerNorm2d>(blocks["norm"]);
+        auto fc1  = std::dynamic_pointer_cast<Conv2d>(blocks["fc1"]);
+        auto conv = std::dynamic_pointer_cast<MoSRDepthwiseConv2d>(blocks["conv"]);
+        auto fc2  = std::dynamic_pointer_cast<Conv2d>(blocks["fc2"]);
+
+        auto h = fc1->forward(ctx, norm->forward(ctx, x));
+
+        auto channel_view = [&](int64_t offset, int64_t count) {
+            return ggml_view_4d(ctx->ggml_ctx, h, h->ne[0], h->ne[1], count, h->ne[3],
+                                h->nb[1], h->nb[2], h->nb[3], offset * h->nb[2]);
+        };
+        auto g = channel_view(0, hidden);
+        auto i = channel_view(hidden, hidden - conv_ch);
+        auto c = conv->forward(ctx, channel_view(2 * hidden - conv_ch, conv_ch));
+        if (hidden > conv_ch) {
+            c = ggml_concat(ctx->ggml_ctx, ggml_ext_cont(ctx->ggml_ctx, i), c, 2);
+        }
+
+        h = ggml_mul(ctx->ggml_ctx, c, mosr_mish(ctx->ggml_ctx, ggml_ext_cont(ctx->ggml_ctx, g)));
+        h = mosr_mish(ctx->ggml_ctx, fc2->forward(ctx, h));
+        return ggml_add(ctx->ggml_ctx, h, ggml_scale_bias(ctx->ggml_ctx, x, 1.f, -0.5f));
+    }
+};
+
+class MoSRConvBlock : public GGMLBlock {
+public:
+    MoSRConvBlock(int in_ch, int out_ch) {
+        blocks["block.0"] = std::shared_ptr<GGMLBlock>(new Conv2d(in_ch, out_ch, {3, 3}, {1, 1}, {1, 1}));
+        blocks["block.2"] = std::shared_ptr<GGMLBlock>(new Conv2d(out_ch, out_ch, {3, 3}, {1, 1}, {1, 1}));
+        blocks["conv11"]  = std::shared_ptr<GGMLBlock>(new Conv2d(in_ch, out_ch, {1, 1}));
+    }
+
+    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+        auto conv0  = std::dynamic_pointer_cast<Conv2d>(blocks["block.0"]);
+        auto conv2  = std::dynamic_pointer_cast<Conv2d>(blocks["block.2"]);
+        auto conv11 = std::dynamic_pointer_cast<Conv2d>(blocks["conv11"]);
+
+        auto out = mosr_mish(ctx->ggml_ctx, conv0->forward(ctx, x));
+        out      = mosr_mish(ctx->ggml_ctx, conv2->forward(ctx, out));
+        return ggml_add(ctx->ggml_ctx, out, conv11->forward(ctx, x));
+    }
+};
+
+class MoSRNet : public GGMLBlock {
+protected:
+    MoSRConfig config;
+
+    static std::string gblock_name(int idx) {
+        return "gblocks." + std::to_string(idx);
+    }
+
+    // torch.nn.PixelShuffle: [n, c*r*r, h, w] -> [n, c, h*r, w*r]
+    static ggml_tensor* pixel_shuffle(ggml_context* ctx, ggml_tensor* x, int r) {
+        const int64_t w = x->ne[0];
+        const int64_t h = x->ne[1];
+        const int64_t c = x->ne[2] / (r * r);
+        const int64_t n = x->ne[3];
+        x               = ggml_reshape_4d(ctx, x, w, h, r, r * c * n);       // [n*c*r, r, h, w]
+        x               = ggml_cont(ctx, ggml_permute(ctx, x, 1, 2, 0, 3));  // [n*c*r, h, w, r]
+        x               = ggml_reshape_4d(ctx, x, r * w, h, r, c * n);       // [n*c, r, h, w*r]
+        x               = ggml_cont(ctx, ggml_permute(ctx, x, 0, 2, 1, 3));  // [n*c, h, r, w*r]
+        return ggml_reshape_4d(ctx, x, r * w, r * h, c, n);
+    }
+
+public:
+    explicit MoSRNet(MoSRConfig config)
+        : config(std::move(config)) {
+        const int dim = this->config.dim;
+        const int nb  = this->config.n_block;
+
+        blocks[gblock_name(0)] = std::shared_ptr<GGMLBlock>(new Conv2d(this->config.in_ch, dim, {3, 3}, {1, 1}, {1, 1}));
+        for (int i = 1; i <= nb; i++) {
+            blocks[gblock_name(i)] = std::shared_ptr<GGMLBlock>(new MoSRGatedCNNBlock(dim,
+                                                                                      this->config.hidden,
+                                                                                      this->config.conv_ch,
+                                                                                      this->config.kernel_size));
+        }
+        blocks[gblock_name(nb + 1)] = std::shared_ptr<GGMLBlock>(new Conv2d(dim, dim * 2, {3, 3}, {1, 1}, {1, 1}));
+        blocks[gblock_name(nb + 3)] = std::shared_ptr<GGMLBlock>(new Conv2d(dim * 2, dim, {3, 3}, {1, 1}, {1, 1}));
+        blocks[gblock_name(nb + 5)] = std::shared_ptr<GGMLBlock>(new Conv2d(dim, dim, {1, 1}));
+        blocks["shortcut"]          = std::shared_ptr<GGMLBlock>(new MoSRConvBlock(this->config.in_ch, dim));
+        blocks["upsampler.0"]       = std::shared_ptr<GGMLBlock>(new Conv2d(dim,
+                                                                            this->config.in_ch * this->config.scale * this->config.scale,
+                                                                            {3, 3},
+                                                                            {1, 1},
+                                                                            {1, 1}));
+    }
+
+    ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+        // x: [n, in_ch, h, w]
+        // return: [n, in_ch, h*scale, w*scale]
+        const int nb   = config.n_block;
+        auto conv_in   = std::dynamic_pointer_cast<Conv2d>(blocks[gblock_name(0)]);
+        auto conv_mid1 = std::dynamic_pointer_cast<Conv2d>(blocks[gblock_name(nb + 1)]);
+        auto conv_mid2 = std::dynamic_pointer_cast<Conv2d>(blocks[gblock_name(nb + 3)]);
+        auto conv_out  = std::dynamic_pointer_cast<Conv2d>(blocks[gblock_name(nb + 5)]);
+        auto shortcut  = std::dynamic_pointer_cast<MoSRConvBlock>(blocks["shortcut"]);
+        auto upsampler = std::dynamic_pointer_cast<Conv2d>(blocks["upsampler.0"]);
+
+        auto skip = shortcut->forward(ctx, x);
+        sd::ggml_graph_cut::mark_graph_cut(skip, "mosr.shortcut", "skip");
+
+        auto feat = conv_in->forward(ctx, x);
+        sd::ggml_graph_cut::mark_graph_cut(feat, "mosr.prelude", "feat");
+        for (int i = 1; i <= nb; i++) {
+            auto block = std::dynamic_pointer_cast<MoSRGatedCNNBlock>(blocks[gblock_name(i)]);
+            feat       = block->forward(ctx, feat);
+            sd::ggml_graph_cut::mark_graph_cut(feat, "mosr.gblocks." + std::to_string(i), "feat");
+        }
+        feat = mosr_mish(ctx->ggml_ctx, conv_mid1->forward(ctx, feat));
+        feat = mosr_mish(ctx->ggml_ctx, conv_mid2->forward(ctx, feat));
+        feat = conv_out->forward(ctx, feat);
+        feat = ggml_add(ctx->ggml_ctx, feat, ggml_scale_bias(ctx->ggml_ctx, skip, 1.f, -0.5f));
+        sd::ggml_graph_cut::mark_graph_cut(feat, "mosr.body.out", "feat");
+
+        auto out = pixel_shuffle(ctx->ggml_ctx, upsampler->forward(ctx, feat), config.scale);
+        sd::ggml_graph_cut::mark_graph_cut(out, "mosr.final", "out");
+        return out;
+    }
+};
+
+struct MoSR : public GGMLRunner {
+    MoSRConfig config;
+    std::unique_ptr<MoSRNet> mosr_net;
+
+    MoSR(ggml_backend_t backend,
+         const String2TensorStorage& tensor_storage_map      = {},
+         std::shared_ptr<RunnerWeightManager> weight_manager = nullptr)
+        : GGMLRunner(backend, weight_manager),
+          config(MoSRConfig::detect_from_weights(tensor_storage_map)) {
+        if (!config.is_valid()) {
+            return;
+        }
+        mosr_net = std::make_unique<MoSRNet>(config);
+        mosr_net->init(params_ctx, tensor_storage_map, "");
+    }
+
+    std::string get_desc() override {
+        return "mosr";
+    }
+
+    void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) {
+        if (!mosr_net) {
+            return;
+        }
+
+        mosr_net->get_param_tensors(tensors);
+    }
+
+    ggml_cgraph* build_graph(const sd::Tensor<float>& x_tensor) {
+        if (!mosr_net)
+            return nullptr;
+        constexpr int kGraphNodes = 1 << 16;  // 65k
+        ggml_cgraph* gf           = new_graph_custom(kGraphNodes);
+        ggml_tensor* x            = make_input(x_tensor);
+
+        auto runner_ctx  = get_context();
+        ggml_tensor* out = mosr_net->forward(&runner_ctx, x);
+        ggml_build_forward_expand(gf, out);
+        return gf;
+    }
+
+    sd::Tensor<float> compute(const int n_threads,
+                              const sd::Tensor<float>& x) {
+        auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); };
+        auto result    = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim());
+        return result;
+    }
+};
+
+#endif  // __SD_MODEL_UPSCALER_MOSR_HPP__
diff --git a/src/model/vae/auto_encoder_kl.hpp b/src/model/vae/auto_encoder_kl.hpp
index 115ff34589..3a4733879a 100644
--- a/src/model/vae/auto_encoder_kl.hpp
+++ b/src/model/vae/auto_encoder_kl.hpp
@@ -544,7 +544,7 @@ class AutoEncoderKLModel : public GGMLBlock {
                        const String2TensorStorage& tensor_storage_map = {},
                        const std::string& prefix                      = "")
         : version(version), decode_only(decode_only), use_video_decoder(use_video_decoder) {
-        if (sd_version_is_dit(version)) {
+        if (sd_version_is_dit(version) && version != VERSION_PIXART) {
             if (sd_version_uses_flux2_vae(version)) {
                 dd_config.z_channels = 32;
                 embed_dim            = 32;
@@ -678,7 +678,7 @@ struct AutoEncoderKL : public VAE {
         if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
             scale_factor = 0.18215f;
             shift_factor = 0.f;
-        } else if (sd_version_is_sdxl(version)) {
+        } else if (sd_version_is_sdxl(version) || sd_version_is_pixart(version)) {
             scale_factor = 0.13025f;
             shift_factor = 0.f;
         } else if (sd_version_is_sd3(version)) {
diff --git a/src/model/vae/ltx_audio_vae.hpp b/src/model/vae/ltx_audio_vae.hpp
index d2314b56a7..2ff8528ecc 100644
--- a/src/model/vae/ltx_audio_vae.hpp
+++ b/src/model/vae/ltx_audio_vae.hpp
@@ -396,6 +396,71 @@ namespace LTXV {
         return ggml_reshape_4d(ctx, out, out_time, channels, 1, 1);
     }
 
+    // F32 depthwise conv without im2col or F16 input rounding; nullptr when the backend lacks CONV_2D_DW.
+    static ggml_tensor* depthwise_conv1d_direct(GGMLRunnerContext* runner_ctx,
+                                                ggml_tensor* x,
+                                                ggml_tensor* filter,
+                                                int stride,
+                                                int padding,
+                                                bool reverse) {
+        auto ctx = runner_ctx->ggml_ctx;
+        if (runner_ctx->backend == nullptr || x->type != GGML_TYPE_F32 || x->ne[2] != 1 || x->ne[3] != 1) {
+            return nullptr;
+        }
+        const int64_t time     = x->ne[0];
+        const int64_t channels = x->ne[1];
+        ggml_tensor* f         = ggml_reshape_4d(ctx, filter, filter->ne[0], 1, 1, 1);
+        if (reverse) {
+            f = reverse_1d_filter(ctx, f);
+        }
+        if (f->type != GGML_TYPE_F32) {
+            f = ggml_cast(ctx, f, GGML_TYPE_F32);
+        }
+        auto kernel = repeat_with_vulkan_f32_workaround(runner_ctx->backend, ctx, f, f->ne[0], 1, 1, channels);  // [K, 1, 1, C]
+        auto input  = ggml_reshape_4d(ctx, ggml_ext_cont(ctx, x), time, 1, channels, 1);                        // [W, H=1, C, N=1]
+        auto out    = ggml_conv_2d_dw_direct(ctx, kernel, input, stride, 1, padding, 0, 1, 1);
+        if (!ggml_backend_supports_op(runner_ctx->backend, out)) {
+            return nullptr;
+        }
+        return ggml_reshape_4d(ctx, out, out->ne[0], channels, 1, 1);
+    }
+
+    // depthwise_conv_transpose1d via the direct conv: same zero-stuffing, reversed filter, padding and gain.
+    static ggml_tensor* depthwise_conv_transpose1d_direct(GGMLRunnerContext* runner_ctx,
+                                                          ggml_tensor* x,
+                                                          ggml_tensor* filter,
+                                                          int stride) {
+        auto ctx = runner_ctx->ggml_ctx;
+        if (x->ne[2] != 1 || x->ne[3] != 1 || filter->ne[1] != 1 || filter->ne[2] != 1 || filter->ne[3] != 1) {
+            return nullptr;
+        }
+        const int64_t time        = x->ne[0];
+        const int64_t channels    = x->ne[1];
+        const int64_t kernel_size = filter->ne[0];
+        const int64_t out_time    = (time - 1) * stride + kernel_size;
+
+        auto x_flat = ggml_reshape_3d(ctx, x, 1, time, channels);
+        if (stride > 1) {
+            auto zero_unit = ggml_ext_scale(ctx, x_flat, 0.0f);
+            auto zero_tail = zero_unit;
+            for (int i = 1; i < stride - 1; ++i) {
+                zero_tail = ggml_concat(ctx, zero_tail, zero_unit, 0);
+            }
+            x_flat = ggml_concat(ctx, x_flat, zero_tail, 0);
+        }
+        x_flat   = ggml_reshape_4d(ctx, x_flat, time * stride, channels, 1, 1);
+        auto out = depthwise_conv1d_direct(runner_ctx, x_flat, filter, 1, static_cast<int>(kernel_size - 1), true);
+        if (out == nullptr) {
+            return nullptr;
+        }
+        if (out->ne[0] > out_time) {
+            out = ggml_ext_slice(ctx, out, 0, 0, out_time);
+        }
+        GGML_ASSERT(out->ne[0] == out_time);
+        out = ggml_ext_scale(ctx, out, static_cast<float>(stride));
+        return ggml_reshape_4d(ctx, out, out_time, channels, 1, 1);
+    }
+
     static ggml_tensor* upsample_waveform_hann(GGMLRunnerContext* runner_ctx,
                                                ggml_tensor* waveform,
                                                ggml_tensor* filter,
@@ -667,8 +732,10 @@ namespace LTXV {
         int up_kernel_size   = 12;
         int down_kernel_size = 12;
 
-        explicit Activation1D(int64_t channels)
-            : channels(channels) {
+        bool direct_dw = false;
+
+        explicit Activation1D(int64_t channels, bool direct_dw = false)
+            : channels(channels), direct_dw(direct_dw) {
             blocks["act"] = std::make_shared<SnakeBeta1D>(channels);
         }
 
@@ -690,16 +757,17 @@ namespace LTXV {
             int up_pad_right = up_pad * up_ratio + (up_kernel_size - up_ratio + 1) / 2;
 
             x = replicate_pad_1d(ctx, x, up_pad, up_pad);
-            x = depthwise_conv_transpose1d(ctx, x, up_filter, up_ratio);
-            x = ggml_ext_slice(ctx->ggml_ctx, x, 0, up_pad_left, x->ne[0] - up_pad_right);
+            ggml_tensor* up = direct_dw ? depthwise_conv_transpose1d_direct(ctx, x, up_filter, up_ratio) : nullptr;
+            x               = up != nullptr ? up : depthwise_conv_transpose1d(ctx, x, up_filter, up_ratio);
+            x               = ggml_ext_slice(ctx->ggml_ctx, x, 0, up_pad_left, x->ne[0] - up_pad_right);
 
             x = act->forward(ctx, x);
 
             int down_pad_left  = down_kernel_size / 2 - (down_kernel_size % 2 == 0 ? 1 : 0);
             int down_pad_right = down_kernel_size / 2;
             x                  = replicate_pad_1d(ctx, x, down_pad_left, down_pad_right);
-            x                  = depthwise_conv1d(ctx, x, down_filter, down_ratio, 0);
-            return x;
+            ggml_tensor* down  = direct_dw ? depthwise_conv1d_direct(ctx, x, down_filter, down_ratio, 0, false) : nullptr;
+            return down != nullptr ? down : depthwise_conv1d(ctx, x, down_filter, down_ratio, 0);
         }
     };
 
diff --git a/src/model/vae/ltx_vae.hpp b/src/model/vae/ltx_vae.hpp
index ac22c09b45..f680f20081 100644
--- a/src/model/vae/ltx_vae.hpp
+++ b/src/model/vae/ltx_vae.hpp
@@ -1300,7 +1300,7 @@ struct LTXVideoVAE : public VAE {
             feat_map[feat_idx] = get_cache_tensor_by_name(temporal_feat_cache_name(feat_idx));
         }
 
-        auto runner_ctx  = get_context();
+        auto runner_ctx  = get_context(gf);
         int feat_count   = 0;
         ggml_tensor* out = vae.decode_tiled_chunk(&runner_ctx,
                                                   z,
@@ -1313,8 +1313,7 @@ struct LTXVideoVAE : public VAE {
         for (int feat_idx = 0; feat_idx < feat_count && feat_idx < static_cast<int>(feat_map.size()); ++feat_idx) {
             ggml_tensor* feat_cache = feat_map[static_cast<size_t>(feat_idx)];
             if (feat_cache != nullptr) {
-                cache(temporal_feat_cache_name(static_cast<size_t>(feat_idx)), feat_cache);
-                ggml_build_forward_expand(gf, feat_cache);
+                runner_ctx.persist_cache_tensor(temporal_feat_cache_name(static_cast<size_t>(feat_idx)), feat_cache);
             }
         }
 
diff --git a/src/model/vae/minimax_h3_audio_vae.hpp b/src/model/vae/minimax_h3_audio_vae.hpp
index a4fc144736..c150559477 100644
--- a/src/model/vae/minimax_h3_audio_vae.hpp
+++ b/src/model/vae/minimax_h3_audio_vae.hpp
@@ -2,6 +2,7 @@
 #define __SD_MODEL_VAE_MINIMAX_H3_AUDIO_VAE_HPP__
 
 #include <array>
+#include <cstdlib>
 #include <string>
 #include <vector>
 
@@ -10,6 +11,15 @@
 
 namespace MiniMaxH3 {
 
+    // SD_H3_AUDIO_DIRECT_DW=0: anti-aliased activations go back to the im2col + mat-vec depthwise convs.
+    static bool audio_direct_dw() {
+        static const bool enabled = [] {
+            const char* v = std::getenv("SD_H3_AUDIO_DIRECT_DW");
+            return v == nullptr || v[0] == '\0' || std::atoi(v) != 0;
+        }();
+        return enabled;
+    }
+
     struct AudioSnake1D : public UnaryBlock {
         int64_t channels;
 
@@ -229,9 +239,9 @@ namespace MiniMaxH3 {
             : channels(channels) {
             for (int i = 0; i < 3; ++i) {
                 blocks["activations." + std::to_string(i * 2)] =
-                    std::make_shared<LTXV::Activation1D>(channels);
+                    std::make_shared<LTXV::Activation1D>(channels, audio_direct_dw());
                 blocks["activations." + std::to_string(i * 2 + 1)] =
-                    std::make_shared<LTXV::Activation1D>(channels);
+                    std::make_shared<LTXV::Activation1D>(channels, audio_direct_dw());
                 blocks["convs1." + std::to_string(i)] =
                     std::make_shared<LTXV::Conv1D>(channels,
                                                    channels,
@@ -298,7 +308,7 @@ namespace MiniMaxH3 {
                 }
                 channels = next_channels;
             }
-            blocks["activation_post"] = std::make_shared<LTXV::Activation1D>(channels);
+            blocks["activation_post"] = std::make_shared<LTXV::Activation1D>(channels, audio_direct_dw());
             blocks["conv_post"]       = std::make_shared<LTXV::Conv1D>(channels,
                                                                  1,
                                                                  7,
@@ -363,33 +373,22 @@ namespace MiniMaxH3 {
             auto pre       = std::dynamic_pointer_cast<AudioAttentionProjection>(blocks["pre_block"]);
             auto mean_proj = std::dynamic_pointer_cast<LTXV::Conv1D>(blocks["mean_proj"]);
 
-            // GGML's batched conv1d storage interleaves the stream dimension
-            // with output channels. Subsequent layers then read stereo samples
-            // as adjacent feature channels. Run each mono stream independently,
-            // matching PyTorch's reshape(B*S, 1, samples), and concatenate only
-            // the completed normalized latents.
             const int64_t streams = waveform->ne[2] * waveform->ne[3];
             waveform              = ggml_reshape_3d(ctx->ggml_ctx,
                                                     waveform,
                                                     waveform->ne[0],
                                                     1,
                                                     streams);
-            ggml_tensor* stereo_z = nullptr;
-            for (int64_t stream = 0; stream < streams; ++stream) {
-                auto mono = ggml_ext_slice(ctx->ggml_ctx, waveform, 2, stream, stream + 1);
-                auto x    = encoder->forward(ctx, mono);
-                x         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
-                x         = pre->forward(ctx, x);
-                x         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
-                auto z    = mean_proj->forward(ctx, x);
-
-                auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
-                auto std  = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
-                z         = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
-                z         = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
-                stereo_z  = stereo_z == nullptr ? z : ggml_concat(ctx->ggml_ctx, stereo_z, z, 1);
-            }
-            return stereo_z;
+            auto x                = encoder->forward(ctx, waveform);
+            x                     = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
+            x                     = pre->forward(ctx, x);
+            x                     = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 1, 0, 2, 3));
+            auto z                = mean_proj->forward(ctx, x);
+
+            auto mean = ggml_reshape_4d(ctx->ggml_ctx, params["latents_mean"], 1, kLatentChannels, 1, 1);
+            auto std  = ggml_reshape_4d(ctx->ggml_ctx, params["latents_std"], 1, kLatentChannels, 1, 1);
+            z         = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, z, mean), std);
+            return ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 0, 2, 1, 3));
         }
 
         ggml_tensor* decode(GGMLRunnerContext* ctx, ggml_tensor* latent) {
diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp
index 777566438c..2b1085d6be 100644
--- a/src/model/vae/minimax_h3_vae.hpp
+++ b/src/model/vae/minimax_h3_vae.hpp
@@ -1,11 +1,15 @@
 #ifndef __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__
 #define __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__
 
+#include <thread>
 #include <algorithm>
 #include <array>
 #include <cmath>
+#include <cstdlib>
+#include <cstring>
 #include <memory>
 #include <string>
+#include <system_error>
 #include <tuple>
 #include <utility>
 #include <vector>
@@ -13,6 +17,7 @@
 #include "model/common/rope.hpp"
 #include "model/diffusion/dit.hpp"
 #include "model/vae/vae.hpp"
+#include "runtime/tiling.h"
 
 namespace MiniMaxH3VAE {
 
@@ -228,11 +233,12 @@ namespace MiniMaxH3VAE {
         return ggml_reshape_3d(ctx, x, x->ne[0], x->ne[1], x->ne[2] * x->ne[3]);
     }
 
-    static ggml_tensor* apply_partial_rope(ggml_context* ctx,
+    static ggml_tensor* apply_partial_rope(GGMLRunnerContext* runner_ctx,
                                            ggml_tensor* x,
                                            ggml_tensor* pe) {
+        auto ctx        = runner_ctx->ggml_ctx;
         int64_t rot_dim = pe->ne[2] * 2;
-        auto rotated    = Rope::apply_rope(ctx,
+        auto rotated    = Rope::apply_rope(runner_ctx,
                                            ggml_ext_slice(ctx, x, 0, 0, rot_dim),
                                            pe,
                                            false);
@@ -244,6 +250,37 @@ namespace MiniMaxH3VAE {
         return ggml_concat(ctx, rotated, tail, 0);
     }
 
+    // Same arithmetic as apply_partial_rope: rotated channel i of each half is
+    // x_lo[i] * pe[0][j][i] + x_hi[i] * pe[1][j][i] as separate mul/mul/add ops, and the tail is
+    // copied unchanged. The halves are read as [half, L, n_head, N] views of the projection
+    // layout, so the per-half permutes, repeats and the tail permute + concat of the generic
+    // path disappear. rope_a / rope_b are the [head_dim, L] coefficient tables built by
+    // MiniMaxH3VideoVAERunner::build_rope_tables. Returns [head_dim, L, n_head * N].
+    static ggml_tensor* apply_partial_rope_tables(ggml_context* ctx,
+                                                  ggml_tensor* x,
+                                                  ggml_tensor* rope_a,
+                                                  ggml_tensor* rope_b,
+                                                  int64_t rot_dim) {
+        const int64_t half = rot_dim / 2;
+        auto heads_view    = [&](int64_t start, int64_t width) {
+            auto view = ggml_view_4d(ctx, x, width, x->ne[1], x->ne[2], x->ne[3], x->nb[1], x->nb[2], x->nb[3], start * x->nb[0]);
+            return ggml_permute(ctx, view, 0, 2, 1, 3);  // [width, L, n_head, N]
+        };
+        auto table = [&](ggml_tensor* t, int64_t start) {
+            return ggml_view_2d(ctx, t, half, t->ne[2], t->nb[2], start * t->nb[0]);  // [half, L]
+        };
+        auto lo  = heads_view(0, half);
+        auto hi  = heads_view(half, half);
+        auto out = ggml_concat(ctx,
+                               ggml_add(ctx, ggml_mul(ctx, lo, table(rope_a, 0)), ggml_mul(ctx, hi, table(rope_b, 0))),
+                               ggml_add(ctx, ggml_mul(ctx, lo, table(rope_b, half)), ggml_mul(ctx, hi, table(rope_a, half))),
+                               0);
+        if (rot_dim < x->ne[0]) {
+            out = ggml_concat(ctx, out, ggml_cont(ctx, heads_view(rot_dim, x->ne[0] - rot_dim)), 0);
+        }
+        return ggml_reshape_3d(ctx, out, out->ne[0], out->ne[1], out->ne[2] * out->ne[3]);
+    }
+
     struct DecoderAttention : public GGMLBlock {
         static constexpr int num_head = 32;
         static constexpr int head_dim = 64;
@@ -254,52 +291,156 @@ namespace MiniMaxH3VAE {
             blocks["to_out"] = std::make_shared<Linear>(dim, dim, true);
         }
 
+        // Single-tile flash-attention decode: one ggml_rope_pe_permute per tensor replaces the table RoPE +
+        // chunk / permute / cast chain, bit-identically. nullptr when it does not apply (SD_H3_VAE_FUSED_QKV=0).
+        ggml_tensor* forward_head_major(GGMLRunnerContext* ctx,
+                                        ggml_tensor* qkv,
+                                        ggml_tensor* pe,
+                                        ggml_tensor* rope_a,
+                                        ggml_tensor* rope_b) {
+#ifndef SD_GGML_H3_FUSED_OPS
+            return nullptr;
+#else
+            static const bool enabled = [] {
+                const char* v = getenv("SD_H3_VAE_FUSED_QKV");
+                return v == nullptr || v[0] == '\0' || atoi(v) != 0;
+            }();
+            // rope_a / rope_b are only passed with SD_H3_VAE_GRAPH_OPT on; that path is what this replaces
+            if (!enabled || rope_a == nullptr || rope_b == nullptr || pe == nullptr || ctx->backend == nullptr ||
+                !ctx->flash_attn_enabled || ctx->sage_attn_enabled || qkv->type != GGML_TYPE_F32 ||
+                qkv->ne[3] != 1 || qkv->nb[0] != sizeof(float)) {
+                return nullptr;
+            }
+            const int64_t sequence = qkv->ne[2];
+            const int n_rot        = static_cast<int>(pe->ne[2] * 2);
+            if (pe->type != GGML_TYPE_F32 || !ggml_is_contiguous(pe) || pe->ne[0] != 2 || pe->ne[1] != 2 ||
+                pe->ne[3] != sequence || n_rot > head_dim) {
+                return nullptr;
+            }
+            const float kv_scale = ctx->attn_scale > 0.f ? ctx->attn_scale : 1.f;
+            auto part            = [&](int index) {
+                return ggml_view_4d(ctx->ggml_ctx, qkv, head_dim, num_head, sequence, 1,
+                                    qkv->nb[1], qkv->nb[2], qkv->nb[3], index * head_dim * sizeof(float));
+            };
+            ggml_tensor* q = nullptr;
+            ggml_tensor* k = nullptr;
+#ifdef SD_GGML_H3_FUSED_QK_NORM
+            // same reduction as GGML_OP_RMS_NORM where supported (SD_H3_VAE_FUSED_QK_NORM=0 keeps it separate)
+            static const bool fuse_norm = [] {
+                const char* v = getenv("SD_H3_VAE_FUSED_QK_NORM");
+                return v == nullptr || v[0] == '\0' || atoi(v) != 0;
+            }();
+            if (fuse_norm) {
+                q = ggml_rope_pe_permute_rms(ctx->ggml_ctx, part(0), pe, n_rot, 1.f, GGML_TYPE_F32, 1e-5f);
+                k = ggml_rope_pe_permute_rms(ctx->ggml_ctx, part(1), pe, n_rot, kv_scale, GGML_TYPE_F16, 1e-5f);
+                if (!ggml_backend_supports_op(ctx->backend, q) || !ggml_backend_supports_op(ctx->backend, k)) {
+                    q = k = nullptr;
+                }
+            }
+#endif
+            if (q == nullptr) {
+                q = ggml_rope_pe_permute(ctx->ggml_ctx, ggml_rms_norm(ctx->ggml_ctx, part(0), 1e-5f), pe, n_rot, 1.f, GGML_TYPE_F32);
+                k = ggml_rope_pe_permute(ctx->ggml_ctx, ggml_rms_norm(ctx->ggml_ctx, part(1), 1e-5f), pe, n_rot, kv_scale, GGML_TYPE_F16);
+            }
+            auto v = ggml_rope_pe_permute(ctx->ggml_ctx, part(2), nullptr, 0, kv_scale, GGML_TYPE_F16);
+            if (!ggml_backend_supports_op(ctx->backend, q) || !ggml_backend_supports_op(ctx->backend, k) ||
+                !ggml_backend_supports_op(ctx->backend, v)) {
+                return nullptr;
+            }
+            q = ggml_reshape_3d(ctx->ggml_ctx, q, head_dim, sequence, num_head);
+            k = ggml_reshape_3d(ctx->ggml_ctx, k, head_dim, sequence, num_head);
+            v = ggml_reshape_3d(ctx->ggml_ctx, v, head_dim, sequence, num_head);
+            return ggml_ext_attention_prepared(ctx->ggml_ctx, ctx->backend, q, k, v, num_head, 1, kv_scale);
+#endif
+        }
+
         ggml_tensor* forward(GGMLRunnerContext* ctx,
                              ggml_tensor* x,
-                             ggml_tensor* pe) {
+                             ggml_tensor* pe,
+                             ggml_tensor* rope_a = nullptr,
+                             ggml_tensor* rope_b = nullptr,
+                             int64_t flat_tiles  = 1) {
             auto to_qkv         = std::dynamic_pointer_cast<Linear>(blocks["to_qkv"]);
             auto to_out         = std::dynamic_pointer_cast<Linear>(blocks["to_out"]);
             auto qkv_projection = to_qkv->forward(ctx, x);
-            int64_t sequence    = x->ne[1];
-            int64_t batch_size  = x->ne[2] * x->ne[3];
-            qkv_projection      = ggml_reshape_4d(ctx->ggml_ctx,
-                                                  qkv_projection,
-                                                  3 * head_dim,
-                                                  num_head,
-                                                  sequence,
-                                                  batch_size);
-            auto qkv            = ggml_ext_chunk(ctx->ggml_ctx, qkv_projection, 3, 0);
-            auto q              = ggml_reshape_4d(ctx->ggml_ctx,
-                                                  qkv[0],
-                                                  head_dim,
-                                                  num_head,
-                                                  sequence,
-                                                  batch_size);
-            auto k              = ggml_reshape_4d(ctx->ggml_ctx,
-                                                  qkv[1],
-                                                  head_dim,
-                                                  num_head,
-                                                  sequence,
-                                                  batch_size);
-            auto v              = ggml_reshape_4d(ctx->ggml_ctx,
-                                                  qkv[2],
-                                                  head_dim,
-                                                  num_head,
-                                                  sequence,
-                                                  batch_size);
-            q                   = ggml_rms_norm(ctx->ggml_ctx, q, 1e-5f);
-            k                   = ggml_rms_norm(ctx->ggml_ctx, k, 1e-5f);
-            q                   = apply_partial_rope(ctx->ggml_ctx, q, pe);
-            k                   = apply_partial_rope(ctx->ggml_ctx, k, pe);
-            auto out            = ggml_ext_attention_ext(ctx,
-                                                         q,
-                                                         k,
-                                                         v,
-                                                         num_head,
-                                                         nullptr,
-                                                         true,
-                                                         ctx->flash_attn_enabled);
-            return to_out->forward(ctx, out);
+            int64_t sequence    = x->ne[1] / flat_tiles;
+            int64_t batch_size  = x->ne[2] * x->ne[3] * flat_tiles;
+            auto project_out    = [&](ggml_tensor* attn) {
+                if (flat_tiles > 1) {
+                    attn = ggml_reshape_2d(ctx->ggml_ctx, attn, attn->ne[0], attn->ne[1] * attn->ne[2] * attn->ne[3]);
+                }
+                return to_out->forward(ctx, attn);
+            };
+            qkv_projection = ggml_reshape_4d(ctx->ggml_ctx,
+                                             qkv_projection,
+                                             3 * head_dim,
+                                             num_head,
+                                             sequence,
+                                             batch_size);
+            if (auto fused = forward_head_major(ctx, qkv_projection, pe, rope_a, rope_b)) {
+                return project_out(fused);
+            }
+            auto qkv       = ggml_ext_chunk(ctx->ggml_ctx, qkv_projection, 3, 0);
+            auto q         = ggml_reshape_4d(ctx->ggml_ctx,
+                                             qkv[0],
+                                             head_dim,
+                                             num_head,
+                                             sequence,
+                                             batch_size);
+            auto k         = ggml_reshape_4d(ctx->ggml_ctx,
+                                             qkv[1],
+                                             head_dim,
+                                             num_head,
+                                             sequence,
+                                             batch_size);
+            auto v         = ggml_reshape_4d(ctx->ggml_ctx,
+                                             qkv[2],
+                                             head_dim,
+                                             num_head,
+                                             sequence,
+                                             batch_size);
+            q              = ggml_rms_norm(ctx->ggml_ctx, q, 1e-5f);
+            k              = ggml_rms_norm(ctx->ggml_ctx, k, 1e-5f);
+            if (rope_a != nullptr && rope_b != nullptr) {
+                q = apply_partial_rope_tables(ctx->ggml_ctx, q, rope_a, rope_b, pe->ne[2] * 2);
+                k = apply_partial_rope_tables(ctx->ggml_ctx, k, rope_a, rope_b, pe->ne[2] * 2);
+            } else {
+                q = apply_partial_rope(ctx, q, pe);
+                k = apply_partial_rope(ctx, k, pe);
+            }
+            const int64_t tiles = v->ne[3];
+            if (tiles > 1) {
+                // Attention kernels split their work by the total problem size (flash attention's
+                // stream-k, batched GEMM heuristics), so a batched call can sum in a different order
+                // than a single-tile call. One call per tile keeps each tile identical to the unbatched decode.
+                std::vector<ggml_tensor*> outs;
+                for (int64_t n = 0; n < tiles; ++n) {
+                    auto qn = ggml_view_3d(ctx->ggml_ctx, q, q->ne[0], q->ne[1], num_head, q->nb[1], q->nb[2], n * num_head * q->nb[2]);
+                    auto kn = ggml_view_3d(ctx->ggml_ctx, k, k->ne[0], k->ne[1], num_head, k->nb[1], k->nb[2], n * num_head * k->nb[2]);
+                    auto vn = ggml_view_4d(ctx->ggml_ctx, v, v->ne[0], v->ne[1], v->ne[2], 1, v->nb[1], v->nb[2], v->nb[3], n * v->nb[3]);
+                    outs.push_back(ggml_ext_attention_ext(ctx, qn, kn, vn, num_head, nullptr, true, ctx->flash_attn_enabled));
+                }
+                while (outs.size() > 1) {
+                    std::vector<ggml_tensor*> merged;
+                    for (size_t i = 0; i + 1 < outs.size(); i += 2) {
+                        merged.push_back(ggml_concat(ctx->ggml_ctx, outs[i], outs[i + 1], 2));
+                    }
+                    if (outs.size() % 2 == 1) {
+                        merged.push_back(outs.back());
+                    }
+                    outs = std::move(merged);
+                }
+                return project_out(outs[0]);
+            }
+            auto out = ggml_ext_attention_ext(ctx,
+                                              q,
+                                              k,
+                                              v,
+                                              num_head,
+                                              nullptr,
+                                              true,
+                                              ctx->flash_attn_enabled);
+            return project_out(out);
         }
     };
 
@@ -312,9 +453,13 @@ namespace MiniMaxH3VAE {
             blocks["w2"] = std::make_shared<Linear>(kInnerDim, dim, true);
         }
 
-        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
-            auto w1   = std::dynamic_pointer_cast<Linear>(blocks["w1"]);
-            auto w2   = std::dynamic_pointer_cast<Linear>(blocks["w2"]);
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, bool fused_glu = false) {
+            auto w1 = std::dynamic_pointer_cast<Linear>(blocks["w1"]);
+            auto w2 = std::dynamic_pointer_cast<Linear>(blocks["w2"]);
+            if (fused_glu) {
+                // silu(first half) * second half in one op; same per-element math as silu + mul.
+                return w2->forward(ctx, ggml_swiglu(ctx->ggml_ctx, w1->forward(ctx, x)));
+            }
             auto gate = ggml_ext_chunk(ctx->ggml_ctx, w1->forward(ctx, x), 2, 0);
             return w2->forward(ctx,
                                ggml_mul(ctx->ggml_ctx,
@@ -344,7 +489,10 @@ namespace MiniMaxH3VAE {
 
         ggml_tensor* forward(GGMLRunnerContext* ctx,
                              ggml_tensor* x,
-                             ggml_tensor* pe) {
+                             ggml_tensor* pe,
+                             ggml_tensor* rope_a = nullptr,
+                             ggml_tensor* rope_b = nullptr,
+                             int64_t flat_tiles  = 1) {
             auto norm1 = std::dynamic_pointer_cast<RMSNorm>(blocks["norm1"]);
             auto attn  = std::dynamic_pointer_cast<DecoderAttention>(blocks["attn"]);
             auto norm2 = std::dynamic_pointer_cast<RMSNorm>(blocks["norm2"]);
@@ -352,12 +500,12 @@ namespace MiniMaxH3VAE {
             x          = ggml_add(ctx->ggml_ctx,
                                   x,
                                   ggml_mul(ctx->ggml_ctx,
-                                           attn->forward(ctx, norm1->forward(ctx, x), pe),
+                                           attn->forward(ctx, norm1->forward(ctx, x), pe, rope_a, rope_b, flat_tiles),
                                            params["scale1"]));
             return ggml_add(ctx->ggml_ctx,
                             x,
                             ggml_mul(ctx->ggml_ctx,
-                                     ff->forward(ctx, norm2->forward(ctx, x)),
+                                     ff->forward(ctx, norm2->forward(ctx, x), rope_a != nullptr),
                                      params["scale2"]));
         }
     };
@@ -396,33 +544,61 @@ namespace MiniMaxH3VAE {
 
         ggml_tensor* forward(GGMLRunnerContext* ctx,
                              ggml_tensor* z,
-                             ggml_tensor* pe) {
+                             ggml_tensor* pe,
+                             ggml_tensor* rope_a = nullptr,
+                             ggml_tensor* rope_b = nullptr) {
             int64_t width      = z->ne[0];
             int64_t height     = z->ne[1];
             int64_t num_frames = z->ne[2];
             int64_t batch_size = z->ne[3] / 24;
-            GGML_ASSERT(batch_size == 1);
-
-            z                   = ggml_cont(ctx->ggml_ctx,
-                                            ggml_ext_torch_permute(ctx->ggml_ctx, z, 3, 0, 1, 2));
-            z                   = ggml_reshape_3d(ctx->ggml_ctx,
-                                                  z,
-                                                  24,
-                                                  width * height * num_frames,
-                                                  batch_size);
-            auto x_embedder     = std::dynamic_pointer_cast<Linear>(blocks["x_embedder"]);
-            auto h              = x_embedder->forward(ctx, z);
+            GGML_ASSERT(batch_size * 24 == z->ne[3]);
+
+            if (batch_size == 1) {
+                z = ggml_cont(ctx->ggml_ctx,
+                              ggml_ext_torch_permute(ctx->ggml_ctx, z, 3, 0, 1, 2));
+                z = ggml_reshape_3d(ctx->ggml_ctx,
+                                    z,
+                                    24,
+                                    width * height * num_frames,
+                                    batch_size);
+            } else {
+                // Tiles are stacked along the channel axis, tile-major: [W, H, T, 24 * N].
+                z = ggml_reshape_3d(ctx->ggml_ctx, z, width * height * num_frames, 24, batch_size);
+                z = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, z, 1, 0, 2, 3));
+            }
+            // Batched tiles run every projection as one 2D matmul over all tiles' tokens. A 3D
+            // activation would broadcast the weight through cublasGemmBatchedEx, whose results
+            // did not match the single-tile GEMM.
+            const bool flat = batch_size > 1;
+            auto x_embedder = std::dynamic_pointer_cast<Linear>(blocks["x_embedder"]);
+            ggml_tensor* h  = nullptr;
+            if (flat) {
+                h = x_embedder->forward(ctx, ggml_reshape_2d(ctx->ggml_ctx, z, z->ne[0], z->ne[1] * z->ne[2]));
+                h = ggml_reshape_3d(ctx->ggml_ctx, h, h->ne[0], z->ne[1], z->ne[2]);
+            } else {
+                h = x_embedder->forward(ctx, z);
+            }
             int64_t num_patches = h->ne[1];
-            h                   = ggml_concat(ctx->ggml_ctx, h, params["register_tokens"], 1);
-            auto zero           = ggml_ext_scale(ctx->ggml_ctx,
-                                                 ggml_ext_slice(ctx->ggml_ctx, h, 1, 0, 1),
-                                                 0.f);
-            h                   = ggml_concat(ctx->ggml_ctx, h, zero, 1);
-
+            auto registers      = params["register_tokens"];
+            if (batch_size > 1) {
+                registers = ggml_repeat(ctx->ggml_ctx,
+                                        registers,
+                                        ggml_new_tensor_3d(ctx->ggml_ctx, registers->type, dim, num_register_tokens, batch_size));
+            }
+            h         = ggml_concat(ctx->ggml_ctx, h, registers, 1);
+            auto zero = ggml_ext_scale(ctx->ggml_ctx,
+                                       ggml_ext_slice(ctx->ggml_ctx, h, 1, 0, 1),
+                                       0.f);
+            h         = ggml_concat(ctx->ggml_ctx, h, zero, 1);
+
+            const int64_t tokens = h->ne[1];
+            if (flat) {
+                h = ggml_reshape_2d(ctx->ggml_ctx, h, h->ne[0], tokens * batch_size);
+            }
             for (int i = 0; i < num_layers; ++i) {
                 auto block = std::dynamic_pointer_cast<DecoderBlock>(
                     blocks["transformer_blocks." + std::to_string(i)]);
-                h = block->forward(ctx, h, pe);
+                h = block->forward(ctx, h, pe, rope_a, rope_b, flat ? batch_size : 1);
                 sd::ggml_graph_cut::mark_graph_cut(h,
                                                    "minimax_h3_vae.decoder.blocks." + std::to_string(i),
                                                    "hidden_states");
@@ -431,7 +607,10 @@ namespace MiniMaxH3VAE {
             auto norm_out = std::dynamic_pointer_cast<LayerNorm>(blocks["norm_out"]);
             auto proj_out = std::dynamic_pointer_cast<Linear>(blocks["proj_out"]);
             h             = proj_out->forward(ctx, norm_out->forward(ctx, h));
-            h             = ggml_ext_slice(ctx->ggml_ctx, h, 1, 0, num_patches);
+            if (flat) {
+                h = ggml_reshape_3d(ctx->ggml_ctx, h, h->ne[0], tokens, batch_size);
+            }
+            h = ggml_ext_slice(ctx->ggml_ctx, h, 1, 0, num_patches);
             return DiT::unpatchify_3d(ctx->ggml_ctx,
                                       h,
                                       num_frames,
@@ -448,8 +627,8 @@ namespace MiniMaxH3VAE {
         MiniMaxH3VideoVAE() {
             blocks["encoder"]         = std::make_shared<Encoder>();
             blocks["quant_conv"]      = std::make_shared<Conv3d>(48,
-                                                            48,
-                                                            std::tuple{1, 1, 1});
+                                                                 48,
+                                                                 std::tuple{1, 1, 1});
             blocks["post_quant_conv"] = std::make_shared<Conv3d>(24,
                                                                  24,
                                                                  std::tuple{1, 1, 1});
@@ -473,10 +652,12 @@ namespace MiniMaxH3VAE {
                             ggml_tensor* latent,
                             ggml_tensor* pe,
                             ggml_tensor* pixel_mean,
-                            ggml_tensor* pixel_std) {
+                            ggml_tensor* pixel_std,
+                            ggml_tensor* rope_a = nullptr,
+                            ggml_tensor* rope_b = nullptr) {
             auto post_quant = std::dynamic_pointer_cast<Conv3d>(blocks["post_quant_conv"]);
             auto decoder    = std::dynamic_pointer_cast<Decoder>(blocks["decoder"]);
-            auto pixels     = decoder->forward(ctx, post_quant->forward(ctx, latent), pe);
+            auto pixels     = decoder->forward(ctx, post_quant->forward(ctx, latent), pe, rope_a, rope_b);
             pixels          = ggml_add(ctx->ggml_ctx,
                                        ggml_mul(ctx->ggml_ctx, pixels, pixel_std),
                                        pixel_mean);
@@ -491,6 +672,9 @@ namespace MiniMaxH3VAE {
         sd::Tensor<float> latents_mean;
         sd::Tensor<float> latents_std;
         sd::Tensor<float> rope_cache;
+        sd::Tensor<float> rope_a_cache;
+        sd::Tensor<float> rope_b_cache;
+        std::array<int64_t, 4> rope_key_ = {0, 0, 0, 0};
 
         MiniMaxH3VideoVAERunner(ggml_backend_t backend,
                                 const String2TensorStorage& tensor_storage_map,
@@ -556,12 +740,27 @@ namespace MiniMaxH3VAE {
                                    tensor.shape()[3]});
         }
 
-        static sd_tiling_params_t h3_tiling(sd_tiling_params_t params) {
+        sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const override {
+            if (!params.enabled) {
+                params.target_overlap = 0.25f;
+            }
+            if (params.tile_size_w == 0 && params.rel_size_w == 0.f) {
+                params.tile_size_w = 256;
+            }
+            if (params.tile_size_h == 0 && params.rel_size_h == 0.f) {
+                params.tile_size_h = 256;
+            }
             params.enabled         = true;
             params.temporal_tiling = false;
-            params.tile_size_x     = 16;
-            params.tile_size_y     = 16;
-            params.target_overlap  = 0.25f;
+            // SD_H3_VAE_TILE=N: opt-in N x N latent tiles (N * 16 image pixels); the seams move, so frames differ
+            // from the default
+            if (const char* tile = getenv("SD_H3_VAE_TILE")) {
+                const int n = atoi(tile);
+                if (n >= 8) {
+                    params.tile_size_w = params.tile_size_h = n * 16;
+                    params.rel_size_w = params.rel_size_h = 0.f;
+                }
+            }
             return params;
         }
 
@@ -576,27 +775,31 @@ namespace MiniMaxH3VAE {
         }
 
         static sd::Tensor<float> blend_temporal(const sd::Tensor<float>& previous,
-                                                const sd::Tensor<float>& current,
+                                                sd::Tensor<float> current,
                                                 int64_t extent) {
-            auto output            = current;
             extent                 = std::min({extent, previous.shape()[2], current.shape()[2]});
             int64_t previous_start = previous.shape()[2] - extent;
+            GGML_ASSERT(previous.dim() == 5 && current.dim() == 5 && previous.shape()[0] == current.shape()[0] &&
+                        previous.shape()[1] == current.shape()[1] && previous.shape()[3] == current.shape()[3] &&
+                        previous.shape()[4] == current.shape()[4]);
+            const int64_t plane           = current.shape()[0] * current.shape()[1];
+            const int64_t previous_frames = previous.shape()[2];
+            const int64_t current_frames  = current.shape()[2];
             for (int64_t b = 0; b < current.shape()[4]; ++b) {
                 for (int64_t c = 0; c < current.shape()[3]; ++c) {
+                    const int64_t channel = b * current.shape()[3] + c;
                     for (int64_t t = 0; t < extent; ++t) {
-                        float wb = static_cast<float>(t) / extent;
-                        float wa = 1.f - wb;
-                        for (int64_t h = 0; h < current.shape()[1]; ++h) {
-                            for (int64_t w = 0; w < current.shape()[0]; ++w) {
-                                output.index(w, h, t, c, b) =
-                                    previous.index(w, h, previous_start + t, c, b) * wa +
-                                    current.index(w, h, t, c, b) * wb;
-                            }
+                        float wb        = static_cast<float>(t) / extent;
+                        float wa        = 1.f - wb;
+                        const float* pa = previous.data() + (channel * previous_frames + previous_start + t) * plane;
+                        float* pb       = current.data() + (channel * current_frames + t) * plane;
+                        for (int64_t i = 0; i < plane; ++i) {
+                            pb[i] = pa[i] * wa + pb[i] * wb;
                         }
                     }
                 }
             }
-            return output;
+            return current;
         }
 
         sd::Tensor<float> encode(int n_threads,
@@ -605,7 +808,7 @@ namespace MiniMaxH3VAE {
                                  bool circular_x = false,
                                  bool circular_y = false) override {
             auto input  = ensure_video_shape(x);
-            auto tiling = h3_tiling(tiling_params);
+            auto tiling = resolve_tiling_params(tiling_params);
             if (input.shape()[2] == 1) {
                 auto encoded = VAE::encode(n_threads, input, tiling, circular_x, circular_y);
                 if (!encoded.empty() && encoded.shape()[2] > 1) {
@@ -627,8 +830,11 @@ namespace MiniMaxH3VAE {
             auto plan   = make_vae_temporal_tile_plan(input.shape()[2], {17, 0});
             auto result = process_vae_temporal_tiles(input, plan, [&](const sd::Tensor<float>& chunk, const VAETemporalTile& tile) {
                 SD_UNUSED(tile);
-                return VAE::encode(n_threads, chunk, tiling, circular_x, circular_y);
+                // keep the runner alive across chunks; ending it here would
+                // evict and reload the encoder weights every chunk
+                return VAE::encode(n_threads, chunk, tiling, circular_x, circular_y, false);
             });
+            runner_end();
             if (result.empty()) {
                 return {};
             }
@@ -646,7 +852,7 @@ namespace MiniMaxH3VAE {
                                  bool circular_y   = false,
                                  bool silent       = false) override {
             auto input  = ensure_video_shape(x);
-            auto tiling = h3_tiling(tiling_params);
+            auto tiling = resolve_tiling_params(tiling_params);
             if (input.shape()[2] == 1) {
                 auto decoded = VAE::decode(n_threads,
                                            input,
@@ -688,25 +894,27 @@ namespace MiniMaxH3VAE {
                 input.shape()[2],
                 {static_cast<int>(tokens_per_chunk + token_overlap), static_cast<int>(token_overlap)});
             GGML_ASSERT(plan.tiles.size() == static_cast<size_t>(num_chunks));
-            auto result = process_vae_temporal_tiles(input, plan, [&](const sd::Tensor<float>& chunk, const VAETemporalTile& tile) {
-                auto decoded = VAE::decode(n_threads,
-                                           chunk,
-                                           tiling,
-                                           true,
-                                           circular_x,
-                                           circular_y,
-                                           silent);
-                if (decoded.empty()) {
-                    return sd::Tensor<float>();
+            const bool keep_resident = env_int("SD_H3_VAE_KEEP_RESIDENT", 1) != 0;
+            // SD_H3_VAE_REUSE_MEMQUERY=0 queries the device for every capacity check.
+            set_reuse_device_query(env_int("SD_H3_VAE_REUSE_MEMQUERY", 1) != 0);
+            struct ReuseQueryGuard {
+                GGMLRunner& runner;
+                ~ReuseQueryGuard() {
+                    runner.set_reuse_device_query(false);
                 }
-
+            } reuse_query_guard{*this};
+            tile_batch_              = 0;
+            per_tile_compute_bytes_  = 0;
+            std::vector<sd::Tensor<float>> pieces;
+            // Runs in chunk order, on a worker thread unless SD_H3_VAE_ASYNC_ASSEMBLY=0.
+            auto assemble = [&](const sd::Tensor<float>& decoded, const VAETemporalTile& tile) {
                 int64_t first_end = std::min<int64_t>(frames_per_chunk, decoded.shape()[2]);
                 auto first        = sd::ops::slice(decoded,
                                                    2,
                                                    std::min<int64_t>(frame_pre_padding, first_end),
                                                    first_end);
                 if (!overlap.empty()) {
-                    first   = blend_temporal(overlap, first, frame_overlap);
+                    first   = blend_temporal(overlap, std::move(first), frame_overlap);
                     overlap = {};
                 }
 
@@ -721,19 +929,294 @@ namespace MiniMaxH3VAE {
                     overlap = {};
                 }
                 return first;
-            });
-            if (result.empty()) {
-                return {};
-            }
-
+            };
+            const bool async_assembly = env_int("SD_H3_VAE_ASYNC_ASSEMBLY", 1) != 0;
+            std::thread assembler;
+            bool assembly_failed = false;
             int64_t expected_frames = input.shape()[2] <= 1 ? 1 : ((x.shape()[2] - 2) / 5) * 17 + 5;
             expected_frames         = std::max<int64_t>(1, expected_frames);
+            // keeps the same frames concat_frames + slice would
+            sd::Tensor<float> result;
+            int64_t frame_offset = 0;
+            auto append_piece    = [&](const sd::Tensor<float>& piece) {
+                GGML_ASSERT(piece.dim() == 5);
+                if (result.empty()) {
+                    std::vector<int64_t> shape = piece.shape();
+                    shape[2]                   = expected_frames;
+                    result                     = sd::Tensor<float>(std::move(shape));
+                }
+                GGML_ASSERT(piece.shape()[0] == result.shape()[0] && piece.shape()[1] == result.shape()[1] &&
+                            piece.shape()[3] == result.shape()[3] && piece.shape()[4] == result.shape()[4]);
+                const int64_t piece_frames = piece.shape()[2];
+                const int64_t keep         = std::min<int64_t>(piece_frames, expected_frames - frame_offset);
+                const int64_t plane        = piece.shape()[0] * piece.shape()[1];
+                const int64_t planes       = piece.shape()[3] * piece.shape()[4];
+                for (int64_t p = 0; keep > 0 && p < planes; ++p) {
+                    memcpy(result.data() + (p * expected_frames + frame_offset) * plane,
+                           piece.data() + p * piece_frames * plane,
+                           sizeof(float) * keep * plane);
+                }
+                frame_offset += std::max<int64_t>(keep, 0);
+            };
+            auto run_assembly = [&](const sd::Tensor<float>& decoded, const VAETemporalTile& tile) {
+                try {
+                    if (async_assembly) {
+                        append_piece(assemble(decoded, tile));
+                    } else {
+                        pieces.push_back(assemble(decoded, tile));
+                    }
+                } catch (const std::exception& error) {
+                    LOG_ERROR("MiniMax-H3 video VAE: chunk assembly failed: %s", error.what());
+                    assembly_failed = true;
+                }
+            };
+            auto join_assembler = [&]() {
+                if (assembler.joinable()) {
+                    assembler.join();
+                }
+            };
+            struct AssemblerGuard {
+                std::thread& thread;
+                ~AssemblerGuard() {
+                    if (thread.joinable()) {
+                        thread.join();
+                    }
+                }
+            } assembler_guard{assembler};
+            bool failed = false;
+            for (const auto& tile : plan.tiles) {
+                auto decoded = decode_spatial_tiles(n_threads,
+                                                    sd::ops::slice(input, 2, tile.start, tile.end),
+                                                    tiling,
+                                                    circular_x,
+                                                    circular_y,
+                                                    silent);
+                if (!keep_resident) {
+                    runner_end();
+                }
+                join_assembler();
+                if (decoded.empty() || assembly_failed) {
+                    failed = true;
+                    break;
+                }
+                if (async_assembly && !tile.last) {
+                    auto chunk = std::make_shared<sd::Tensor<float>>(std::move(decoded));
+                    try {
+                        assembler = std::thread([&run_assembly, chunk, tile]() {
+                            run_assembly(*chunk, tile);
+                        });
+                    } catch (const std::system_error&) {
+                        run_assembly(*chunk, tile);  // no thread available: assemble inline
+                    }
+                } else {
+                    run_assembly(decoded, tile);
+                }
+            }
+            join_assembler();
+            runner_end();
+            if (failed || assembly_failed) {
+                return {};
+            }
+            if (async_assembly) {
+                if (result.empty()) {
+                    return {};
+                }
+                if (frame_offset < expected_frames) {
+                    result = sd::ops::slice(result, 2, 0, frame_offset);
+                }
+                return result;
+            }
+            if (pieces.empty()) {
+                return {};
+            }
+            result = concat_frames(pieces);
             if (result.shape()[2] > expected_frames) {
                 result = sd::ops::slice(result, 2, 0, expected_frames);
             }
             return result;
         }
 
+        int tile_batch_                = 0;
+        size_t per_tile_compute_bytes_ = 0;
+
+        static int env_int(const char* name, int fallback) {
+            const char* value = getenv(name);
+            if (value == nullptr || value[0] == '\0') {
+                return fallback;
+            }
+            return atoi(value);
+        }
+
+        static sd::Tensor<float> concat_frames(const std::vector<sd::Tensor<float>>& pieces) {
+            std::vector<int64_t> shape = pieces[0].shape();
+            int64_t frames             = 0;
+            for (const auto& piece : pieces) {
+                GGML_ASSERT(piece.dim() == 5 && piece.shape()[0] == shape[0] && piece.shape()[1] == shape[1] &&
+                            piece.shape()[3] == shape[3] && piece.shape()[4] == shape[4]);
+                frames += piece.shape()[2];
+            }
+            shape[2] = frames;
+            sd::Tensor<float> output(shape);
+            const int64_t plane  = shape[0] * shape[1];
+            const int64_t planes = shape[3] * shape[4];
+            int64_t frame_offset = 0;
+            for (const auto& piece : pieces) {
+                const int64_t piece_frames = piece.shape()[2];
+                for (int64_t p = 0; p < planes; ++p) {
+                    memcpy(output.data() + (p * frames + frame_offset) * plane,
+                           piece.data() + p * piece_frames * plane,
+                           sizeof(float) * piece_frames * plane);
+                }
+                frame_offset += piece_frames;
+            }
+            return output;
+        }
+
+        // How many spatial tiles go into one decoder graph. Default 1 (one graph per tile): the
+        // batched graph runs each projection as one matmul over all tiles' tokens, and cuBLAS may
+        // pick a different kernel for the larger M (seen on RTX PRO 6000 Blackwell: 53 dB PSNR vs
+        // the per-tile decode), while it measured no faster there. SD_H3_VAE_TILE_BATCH=N forces N,
+        // SD_H3_VAE_TILE_BATCH=auto runs the first tile alone, measures its compute buffer and sizes
+        // the batch from the free device memory.
+        int resolve_tile_batch() {
+            const char* mode = getenv("SD_H3_VAE_TILE_BATCH");
+            if (mode == nullptr || mode[0] == '\0') {
+                return 1;
+            }
+            int forced = strcmp(mode, "auto") == 0 ? 0 : atoi(mode);
+            if (forced > 0) {
+                return forced;
+            }
+            if (sd_backend_is_cpu(runtime_backend)) {
+                return 1;
+            }
+            if (per_tile_compute_bytes_ == 0) {
+                return 1;
+            }
+            if (tile_batch_ > 0) {
+                return tile_batch_;
+            }
+            size_t free_bytes      = 0;
+            size_t total_bytes     = 0;
+            ggml_backend_dev_t dev = ggml_backend_get_device(runtime_backend);
+            if (dev == nullptr) {
+                tile_batch_ = 1;
+                return tile_batch_;
+            }
+            ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
+            const size_t reserve   = std::max<size_t>(size_t(1) << 30, total_bytes / 10);
+            const size_t available = free_bytes > reserve ? free_bytes - reserve : 0;
+            // Sized as if the single-tile buffer were still held while the batched one is allocated.
+            int batch   = static_cast<int>(available / per_tile_compute_bytes_);
+            int limit   = std::max(1, env_int("SD_H3_VAE_TILE_BATCH_MAX", 4));
+            tile_batch_ = std::max(1, std::min(batch, limit));
+            LOG_INFO("MiniMax-H3 video VAE: %d tile(s) per decoder graph (%.0f MB per tile, %.0f MB free)",
+                     tile_batch_,
+                     per_tile_compute_bytes_ / (1024.0 * 1024.0),
+                     free_bytes / (1024.0 * 1024.0));
+            return tile_batch_;
+        }
+
+        // Tiles are stacked along the channel axis, so decoder output tile i is the i-th contiguous
+        // block of the [W, H, T, 3 * N, 1] result.
+        TileBatchOutput compute_tile_batch(int n_threads,
+                                           const std::vector<sd::Tensor<float>>& tiles) {
+            TileBatchOutput output;
+            output.stack_dim = 3;
+            if (tiles.size() == 1) {
+                output.data = _compute(n_threads, tiles[0], true);
+                if (per_tile_compute_bytes_ == 0) {
+                    per_tile_compute_bytes_ = reusable_compute_buffer_bytes();
+                }
+                return output;
+            }
+            auto batched = ensure_video_shape(tiles[0]);
+            for (size_t i = 1; i < tiles.size(); ++i) {
+                batched = sd::ops::concat(batched, ensure_video_shape(tiles[i]), 3);
+            }
+            output.data = _compute(n_threads, batched, true);
+            return output;
+        }
+
+        sd::Tensor<float> decode_spatial_tiles(int n_threads,
+                                               const sd::Tensor<float>& chunk,
+                                               const sd_tiling_params_t& tiling,
+                                               bool circular_x,
+                                               bool circular_y,
+                                               bool silent) {
+            const int scale_factor = get_scale_factor();
+            float tile_overlap;
+            int tile_size_w, tile_size_h;
+            if (!get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling, chunk.shape()[0], chunk.shape()[1], circular_x, circular_y)) {
+                return {};
+            }
+            // Same circular handling as VAE::tiled_compute: full-width axes wrap in convolutions, split axes wrap
+            // between tiles.
+            const bool original_circular_x = circular_x_enabled;
+            const bool original_circular_y = circular_y_enabled;
+            circular_x                     = circular_x || original_circular_x;
+            circular_y                     = circular_y || original_circular_y;
+            set_circular_axes(circular_x && tile_size_w >= chunk.shape()[0],
+                              circular_y && tile_size_h >= chunk.shape()[1]);
+            struct CircularAxesGuard {
+                VAE& vae;
+                bool x;
+                bool y;
+                ~CircularAxesGuard() {
+                    vae.set_circular_axes(x, y);
+                }
+            } circular_axes_guard{*this, original_circular_x, original_circular_y};
+            return process_tiles_2d_batched(
+                chunk,
+                static_cast<int>(chunk.shape()[0] * scale_factor),
+                static_cast<int>(chunk.shape()[1] * scale_factor),
+                scale_factor,
+                tile_size_w,
+                tile_size_h,
+                tile_overlap,
+                circular_x && tile_size_w < chunk.shape()[0],
+                circular_y && tile_size_h < chunk.shape()[1],
+                [&](int) { return resolve_tile_batch(); },
+                [&](const std::vector<sd::Tensor<float>>& tiles) {
+                    auto output = compute_tile_batch(n_threads, tiles);
+                    if (output.data.empty()) {
+                        LOG_ERROR("vae decode compute failed while processing a tile");
+                    }
+                    return output;
+                },
+                silent);
+        }
+
+        // rope_cache is [2, 2, half, L] = [[cos, -sin], [sin, cos]] per (pair, position). Rotated
+        // output d = i + half * j is x_lo[i] * pe[0][j][i] + x_hi[i] * pe[1][j][i]; tables hold the
+        // coefficient of x[d] (a) and of its partner x[d +- half] (b) for every head channel d.
+        void build_rope_tables() {
+            constexpr int64_t head_dim = DecoderAttention::head_dim;
+            const int64_t half         = rope_cache.shape()[2];
+            const int64_t positions    = rope_cache.shape()[3];
+            GGML_ASSERT(rope_cache.shape()[0] == 2 && rope_cache.shape()[1] == 2 && half * 2 <= head_dim);
+            rope_a_cache    = sd::Tensor<float>({head_dim, 1, positions});
+            rope_b_cache    = sd::Tensor<float>({head_dim, 1, positions});
+            const float* pe = rope_cache.data();
+            auto pe_at      = [&](int64_t a, int64_t j, int64_t i, int64_t l) {
+                return pe[((l * half + i) * 2 + j) * 2 + a];
+            };
+            for (int64_t l = 0; l < positions; ++l) {
+                float* a = rope_a_cache.data() + l * head_dim;
+                float* b = rope_b_cache.data() + l * head_dim;
+                for (int64_t i = 0; i < half; ++i) {
+                    a[i]        = pe_at(0, 0, i, l);
+                    b[i]        = pe_at(1, 0, i, l);
+                    a[half + i] = pe_at(1, 1, i, l);
+                    b[half + i] = pe_at(0, 1, i, l);
+                }
+                for (int64_t d = half * 2; d < head_dim; ++d) {
+                    a[d] = 1.f;
+                    b[d] = 0.f;
+                }
+            }
+        }
+
         sd::Tensor<float> build_rope(int64_t width,
                                      int64_t height,
                                      int64_t num_frames) {
@@ -767,11 +1250,19 @@ namespace MiniMaxH3VAE {
         sd::Tensor<float> _compute(const int n_threads,
                                    const sd::Tensor<float>& z,
                                    bool decode_graph) override {
-            auto input = ensure_video_shape(z);
+            auto input           = ensure_video_shape(z);
+            const bool graph_opt = decode_graph && env_int("SD_H3_VAE_GRAPH_OPT", 1) != 0;
             if (decode_graph) {
-                rope_cache = build_rope(input.shape()[0],
-                                        input.shape()[1],
-                                        input.shape()[2]);
+                const std::array<int64_t, 4> rope_key = {input.shape()[0], input.shape()[1], input.shape()[2], graph_opt ? 1 : 0};
+                if (rope_key != rope_key_ || rope_cache.empty()) {
+                    rope_cache = build_rope(input.shape()[0],
+                                            input.shape()[1],
+                                            input.shape()[2]);
+                    if (graph_opt) {
+                        build_rope_tables();
+                    }
+                    rope_key_ = rope_key;
+                }
             }
             auto get_graph = [&]() -> ggml_cgraph* {
                 auto value       = make_input(input);
@@ -780,8 +1271,10 @@ namespace MiniMaxH3VAE {
                 auto runner_ctx  = get_context();
                 ggml_tensor* out = nullptr;
                 if (decode_graph) {
-                    auto pe = make_input(rope_cache);
-                    out     = model.decode(&runner_ctx, value, pe, mean, std);
+                    auto pe            = make_input(rope_cache);
+                    ggml_tensor* ropea = graph_opt ? make_input(rope_a_cache) : nullptr;
+                    ggml_tensor* ropeb = graph_opt ? make_input(rope_b_cache) : nullptr;
+                    out                = model.decode(&runner_ctx, value, pe, mean, std, ropea, ropeb);
                 } else {
                     out = model.encode(&runner_ctx, value, mean, std);
                 }
diff --git a/src/model/vae/tae.hpp b/src/model/vae/tae.hpp
index e9257b7c82..a0700cab34 100644
--- a/src/model/vae/tae.hpp
+++ b/src/model/vae/tae.hpp
@@ -84,12 +84,17 @@ class TinyEncoder : public UnaryBlock {
     int channels    = 64;
     int z_channels  = 4;
     int num_blocks  = 3;
+    bool f16;
 
 public:
-    TinyEncoder(int z_channels = 4, bool use_midblock_gn = false)
-        : z_channels(z_channels) {
-        int index                       = 0;
+    TinyEncoder(int z_channels = 4, bool use_midblock_gn = false, bool f16 = false)
+        : z_channels(z_channels), f16(f16) {
+        in_channels                     = f16 ? 16 : 3;
+        int index                       = f16 ? 1 : 0;
         blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(in_channels, channels, {3, 3}, {1, 1}, {1, 1}));
+        if (f16) {
+            index++;  // nn.ReLU()
+        }
         blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new TAEBlock(channels, channels));
 
         blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false));
@@ -97,12 +102,14 @@ class TinyEncoder : public UnaryBlock {
             blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new TAEBlock(channels, channels));
         }
 
-        blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false));
+        blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, f16 ? channels * 2 : channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false));
+        channels *= f16 ? 2 : 1;
         for (int i = 0; i < num_blocks; i++) {
             blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new TAEBlock(channels, channels));
         }
 
-        blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false));
+        blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, f16 ? channels * 2 : channels, {3, 3}, {2, 2}, {1, 1}, {1, 1}, false));
+        channels *= f16 ? 2 : 1;
         for (int i = 0; i < num_blocks; i++) {
             blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new TAEBlock(channels, channels, use_midblock_gn));
         }
@@ -114,7 +121,11 @@ class TinyEncoder : public UnaryBlock {
         // x: [n, in_channels, h, w]
         // return: [n, z_channels, h/8, w/8]
 
-        for (int i = 0; i < num_blocks * 3 + 6; i++) {
+        for (int i = f16 ? 1 : 0; i < num_blocks * 3 + 6 + (f16 ? 2 : 0); i++) {
+            if (f16 && i == 2) {
+                x = ggml_relu_inplace(ctx->ggml_ctx, x);
+                continue;
+            }
             auto block = std::dynamic_pointer_cast<UnaryBlock>(blocks[std::to_string(i)]);
 
             x = block->forward(ctx, x);
@@ -131,9 +142,11 @@ class TinyDecoder : public UnaryBlock {
     int num_blocks   = 3;
 
 public:
-    TinyDecoder(int z_channels = 4, bool use_midblock_gn = false)
+    TinyDecoder(int z_channels = 4, bool use_midblock_gn = false, bool f16 = false)
         : z_channels(z_channels) {
-        int index = 0;
+        channels     = f16 ? 256 : 64;
+        out_channels = f16 ? 16 : 3;
+        int index    = 0;
 
         blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(z_channels, channels, {3, 3}, {1, 1}, {1, 1}));
         index++;  // nn.ReLU()
@@ -142,13 +155,15 @@ class TinyDecoder : public UnaryBlock {
             blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new TAEBlock(channels, channels, use_midblock_gn));
         }
         index++;  // nn.Upsample()
-        blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, channels, {3, 3}, {1, 1}, {1, 1}, {1, 1}, false));
+        blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, f16 ? channels / 2 : channels, {3, 3}, {1, 1}, {1, 1}, {1, 1}, false));
+        channels /= f16 ? 2 : 1;
 
         for (int i = 0; i < num_blocks; i++) {
             blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new TAEBlock(channels, channels));
         }
         index++;  // nn.Upsample()
-        blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, channels, {3, 3}, {1, 1}, {1, 1}, {1, 1}, false));
+        blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new Conv2d(channels, f16 ? channels / 2 : channels, {3, 3}, {1, 1}, {1, 1}, {1, 1}, false));
+        channels /= f16 ? 2 : 1;
 
         for (int i = 0; i < num_blocks; i++) {
             blocks[std::to_string(index++)] = std::shared_ptr<GGMLBlock>(new TAEBlock(channels, channels));
@@ -691,6 +706,7 @@ class TAESD : public GGMLBlock {
 protected:
     bool decode_only;
     bool taef2 = false;
+    bool f16   = false;
 
 public:
     int z_channels = 4;
@@ -701,17 +717,21 @@ class TAESD : public GGMLBlock {
         bool use_midblock_gn = false;
         taef2                = sd_version_uses_flux2_vae(version);
 
-        if (sd_version_is_dit(version)) {
+        if (sd_version_is_dit(version) && !sd_version_is_pixart(version)) {
             z_channels = 16;
         }
         if (taef2) {
             z_channels      = 32;
             use_midblock_gn = true;
         }
-        blocks["decoder.layers"] = std::shared_ptr<GGMLBlock>(new TinyDecoder(z_channels, use_midblock_gn));
+        f16 = version == VERSION_QWEN_IMAGE_2_1;
+        if (f16) {
+            z_channels = 64;
+        }
+        blocks["decoder.layers"] = std::shared_ptr<GGMLBlock>(new TinyDecoder(z_channels, use_midblock_gn, f16));
 
         if (!decode_only) {
-            blocks["encoder.layers"] = std::shared_ptr<GGMLBlock>(new TinyEncoder(z_channels, use_midblock_gn));
+            blocks["encoder.layers"] = std::shared_ptr<GGMLBlock>(new TinyEncoder(z_channels, use_midblock_gn, f16));
         }
     }
 
@@ -720,10 +740,15 @@ class TAESD : public GGMLBlock {
         if (taef2) {
             z = unpatchify(ctx->ggml_ctx, z, 2);
         }
-        return decoder->forward(ctx, z);
+        auto x = decoder->forward(ctx, z);
+        return f16 ? unpatchify(ctx->ggml_ctx, x, 2) : x;
     }
 
     ggml_tensor* encode(GGMLRunnerContext* ctx, ggml_tensor* x) {
+        if (f16) {
+            GGML_ASSERT(x->ne[0] % 2 == 0 && x->ne[1] % 2 == 0);
+            x = patchify(ctx->ggml_ctx, x, 2);
+        }
         auto encoder = std::dynamic_pointer_cast<TinyEncoder>(blocks["encoder.layers"]);
         auto z       = encoder->forward(ctx, x);
         if (taef2) {
diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp
index 4d8d13517b..a6f9457ba8 100644
--- a/src/model/vae/vae.hpp
+++ b/src/model/vae/vae.hpp
@@ -1,6 +1,9 @@
 #ifndef __SD_MODEL_VAE_VAE_HPP__
 #define __SD_MODEL_VAE_VAE_HPP__
 
+#include <cmath>
+#include <limits>
+
 #include "core/tensor_ggml.hpp"
 #include "model/common/block.hpp"
 #include "model/vae/vae_tiling.hpp"
@@ -117,8 +120,8 @@ struct VAE : public GGMLRunner {
                                     int output_width,
                                     int output_height,
                                     int scale,
-                                    int p_tile_size_x,
-                                    int p_tile_size_y,
+                                    int p_tile_size_w,
+                                    int p_tile_size_h,
                                     float tile_overlap_factor,
                                     bool circular_x,
                                     bool circular_y,
@@ -138,20 +141,35 @@ struct VAE : public GGMLRunner {
             }
             return output_tile;
         };
-        return ::process_tiles_2d(input,
-                                  output_width,
-                                  output_height,
-                                  scale,
-                                  p_tile_size_x,
-                                  p_tile_size_y,
-                                  tile_overlap_factor,
-                                  circular_x,
-                                  circular_y,
-                                  on_processing,
-                                  silent);
+        const bool original_circular_x = circular_x_enabled;
+        const bool original_circular_y = circular_y_enabled;
+        const int64_t latent_width     = decode_graph ? input.shape()[0] : output_width;
+        const int64_t latent_height    = decode_graph ? input.shape()[1] : output_height;
+        circular_x                     = circular_x || original_circular_x;
+        circular_y                     = circular_y || original_circular_y;
+        // Full-width axes wrap in convolutions; split axes wrap between tiles.
+        set_circular_axes(circular_x && p_tile_size_w >= latent_width,
+                          circular_y && p_tile_size_h >= latent_height);
+        auto output = ::process_tiles_2d(input,
+                                         output_width,
+                                         output_height,
+                                         scale,
+                                         p_tile_size_w,
+                                         p_tile_size_h,
+                                         tile_overlap_factor,
+                                         circular_x && p_tile_size_w < latent_width,
+                                         circular_y && p_tile_size_h < latent_height,
+                                         on_processing,
+                                         silent);
+        set_circular_axes(original_circular_x, original_circular_y);
+        return output;
     }
 
 public:
+    // OOM-retry cap on latent tile sizes (0 = none).
+    int retry_tile_limit_w = 0;
+    int retry_tile_limit_h = 0;
+
     VAE(SDVersion version,
         ggml_backend_t backend,
         const std::string& weight_prefix                    = "",
@@ -162,11 +180,20 @@ struct VAE : public GGMLRunner {
         int scale_factor = 8;
         if (version == VERSION_LTXAV) {
             scale_factor = 32;
-        } else if (version == VERSION_WAN2_2_TI2V || version == VERSION_QWEN_IMAGE_2_1 || sd_version_is_hunyuan_video(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) {
+        } else if (version == VERSION_WAN2_2_TI2V ||
+                   version == VERSION_QWEN_IMAGE_2_1 ||
+                   sd_version_is_hunyuan_video(version) ||
+                   sd_version_is_mage_flow(version) ||
+                   sd_version_is_minimax_h3(version)) {
             scale_factor = 16;
         } else if (sd_version_uses_flux2_vae(version)) {
             scale_factor = 16;
-        } else if (version == VERSION_CHROMA_RADIANCE || version == VERSION_HIDREAM_O1 || sd_version_is_minit2i(version) || sd_version_is_sensenova_u1(version)) {
+        } else if (version == VERSION_CHROMA_RADIANCE ||
+                   version == VERSION_HIDREAM_O1 ||
+                   sd_version_is_minit2i(version) ||
+                   sd_version_is_sensenova_u1(version) ||
+                   sd_version_is_z_image_l2p(version) ||
+                   version == VERSION_IRIS) {
             scale_factor = 1;
         }
         return scale_factor;
@@ -178,33 +205,66 @@ struct VAE : public GGMLRunner {
         return supports_temporal_tiling(VAETemporalDirection::DECODE);
     }
 
-    void get_tile_sizes(int& tile_size_x,
-                        int& tile_size_y,
+    virtual sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const {
+        return params;
+    }
+
+    bool get_tile_sizes(int& tile_size_w,
+                        int& tile_size_h,
                         float& tile_overlap,
                         const sd_tiling_params_t& params,
-                        int64_t latent_x,
-                        int64_t latent_y,
-                        float encoding_factor = 1.0f) {
-        tile_overlap       = std::max(std::min(params.target_overlap, 0.5f), 0.0f);
-        auto get_tile_size = [&](int requested_size, float factor, int64_t latent_size) {
-            const int default_tile_size  = 32;
-            const int min_tile_dimension = 4;
-            int tile_size                = default_tile_size;
-            // factor <= 1 means simple fraction of the latent dimension
-            // factor > 1 means number of tiles across that dimension
-            if (factor > 0.f) {
-                if (factor > 1.0)
-                    factor = 1 / (factor - factor * tile_overlap + tile_overlap);
-                tile_size = static_cast<int>(std::round(latent_size * factor));
-            } else if (requested_size >= min_tile_dimension) {
-                tile_size = requested_size;
+                        int64_t latent_w,
+                        int64_t latent_h,
+                        bool circular_x = false,
+                        bool circular_y = false) {
+        const auto tiling = resolve_tiling_params(params);
+        if (latent_w <= 0 || latent_h <= 0 ||
+            latent_w > std::numeric_limits<int>::max() || latent_h > std::numeric_limits<int>::max() ||
+            !std::isfinite(tiling.target_overlap)) {
+            LOG_ERROR("invalid VAE tiling dimensions or overlap");
+            return false;
+        }
+        const int scale_factor = get_scale_factor();
+        tile_overlap           = std::max(std::min(tiling.target_overlap, 0.5f), 0.0f);
+        auto get_tile_size     = [&](int requested_size, double factor, int64_t latent_size, int& tile_size) {
+            if (requested_size < 0 || !std::isfinite(factor) || factor < 0.0) {
+                LOG_ERROR("VAE tile sizes and relative sizes must be finite and non-negative");
+                return false;
             }
-            tile_size = static_cast<int>(tile_size * encoding_factor);
-            return std::max(std::min(tile_size, static_cast<int>(latent_size)), min_tile_dimension);
+            const int min_tile_dimension = std::min(4, static_cast<int>(latent_size));
+            double size                  = (requested_size > 0 ? requested_size : 256) / scale_factor;
+            if (factor > 0.0) {
+                if (factor > 1.0) {
+                    factor = 1.0 / (factor * (1.0 - tile_overlap) + tile_overlap);
+                }
+                size = std::floor(static_cast<double>(latent_size) * factor);
+            }
+            if (size < min_tile_dimension && (requested_size > 0 || factor > 0.0)) {
+                LOG_ERROR("VAE tile size must be at least %d image pixels on this axis", min_tile_dimension * scale_factor);
+                return false;
+            }
+            tile_size = static_cast<int>(std::min(static_cast<double>(latent_size), std::max<double>(min_tile_dimension, size)));
+            return true;
         };
 
-        tile_size_x = get_tile_size(params.tile_size_x, params.rel_size_x, latent_x);
-        tile_size_y = get_tile_size(params.tile_size_y, params.rel_size_y, latent_y);
+        if (!get_tile_size(tiling.tile_size_w, tiling.rel_size_w, latent_w, tile_size_w) ||
+            !get_tile_size(tiling.tile_size_h, tiling.rel_size_h, latent_h, tile_size_h)) {
+            return false;
+        }
+        // Circular axes wrap their tiles instead of pinning the last one to the edge, so they keep the requested size.
+        if (!circular_x && !circular_x_enabled) {
+            tile_size_w = sd_tiling_seam_safe_tile_size(static_cast<int>(latent_w), tile_size_w, tile_overlap,
+                                                        retry_tile_limit_w);
+        } else if (retry_tile_limit_w > 0) {
+            tile_size_w = std::min(tile_size_w, std::max(4, retry_tile_limit_w));
+        }
+        if (!circular_y && !circular_y_enabled) {
+            tile_size_h = sd_tiling_seam_safe_tile_size(static_cast<int>(latent_h), tile_size_h, tile_overlap,
+                                                        retry_tile_limit_h);
+        } else if (retry_tile_limit_h > 0) {
+            tile_size_h = std::min(tile_size_h, std::max(4, retry_tile_limit_h));
+        }
+        return true;
     }
 
     virtual sd::Tensor<float> encode(int n_threads,
@@ -212,7 +272,17 @@ struct VAE : public GGMLRunner {
                                      sd_tiling_params_t tiling_params,
                                      bool circular_x = false,
                                      bool circular_y = false) {
+        return encode(n_threads, x, tiling_params, circular_x, circular_y, true);
+    }
+
+    sd::Tensor<float> encode(int n_threads,
+                             const sd::Tensor<float>& x,
+                             sd_tiling_params_t tiling_params,
+                             bool circular_x,
+                             bool circular_y,
+                             bool auto_end) {
         int64_t t0              = ggml_time_ms();
+        tiling_params           = resolve_tiling_params(tiling_params);
         sd::Tensor<float> input = x;
         sd::Tensor<float> output;
         if (scale_input) {
@@ -224,21 +294,19 @@ struct VAE : public GGMLRunner {
             int64_t W              = input.shape()[0] / scale_factor;
             int64_t H              = input.shape()[1] / scale_factor;
             float tile_overlap;
-            int tile_size_x, tile_size_y;
-            // Image VAE encode is more sensitive to tile boundary context than decode.
-            // Keep the smaller legacy factor for video VAEs, but default image encode
-            // tiles to 64 latent pixels so a 512px SD image is encoded as one tile.
-            const float encode_tile_factor = sd_version_is_minimax_h3(version) ? 1.f : (sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version)) ? 1.30539f
-                                                                                                                                                                                            : 2.0f;
-            get_tile_sizes(tile_size_x, tile_size_y, tile_overlap, tiling_params, W, H, encode_tile_factor);
-            LOG_VERBOSE("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
+            int tile_size_w, tile_size_h;
+            if (!get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, W, H, circular_x, circular_y)) {
+                return {};
+            }
+            LOG_VERBOSE("VAE encode tile size: %dx%d pixels (%dx%d latent)",
+                        tile_size_w * scale_factor, tile_size_h * scale_factor, tile_size_w, tile_size_h);
             output = tiled_compute(input,
                                    n_threads,
                                    static_cast<int>(W),
                                    static_cast<int>(H),
                                    scale_factor,
-                                   tile_size_x,
-                                   tile_size_y,
+                                   tile_size_w,
+                                   tile_size_h,
                                    tile_overlap,
                                    circular_x,
                                    circular_y,
@@ -252,7 +320,9 @@ struct VAE : public GGMLRunner {
                                                   tiling_params);
         }
 
-        runner_end();
+        if (auto_end) {
+            runner_end();
+        }
 
         if (output.empty()) {
             LOG_ERROR("vae encode compute failed");
@@ -270,7 +340,19 @@ struct VAE : public GGMLRunner {
                                      bool circular_x   = false,
                                      bool circular_y   = false,
                                      bool silent       = false) {
+        return decode(n_threads, x, tiling_params, decode_video, circular_x, circular_y, silent, true);
+    }
+
+    sd::Tensor<float> decode(int n_threads,
+                             const sd::Tensor<float>& x,
+                             sd_tiling_params_t tiling_params,
+                             bool decode_video,
+                             bool circular_x,
+                             bool circular_y,
+                             bool silent,
+                             bool auto_end) {
         int64_t t0              = ggml_time_ms();
+        tiling_params           = resolve_tiling_params(tiling_params);
         sd::Tensor<float> input = x;
         sd::Tensor<float> output;
 
@@ -279,10 +361,13 @@ struct VAE : public GGMLRunner {
             int64_t W              = input.shape()[0] * scale_factor;
             int64_t H              = input.shape()[1] * scale_factor;
             float tile_overlap;
-            int tile_size_x, tile_size_y;
-            get_tile_sizes(tile_size_x, tile_size_y, tile_overlap, tiling_params, input.shape()[0], input.shape()[1]);
+            int tile_size_w, tile_size_h;
+            if (!get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params, input.shape()[0], input.shape()[1], circular_x, circular_y)) {
+                return {};
+            }
             if (!silent) {
-                LOG_VERBOSE("VAE Tile size: %dx%d", tile_size_x, tile_size_y);
+                LOG_VERBOSE("VAE decode tile size: %dx%d pixels (%dx%d latent)",
+                            tile_size_w * scale_factor, tile_size_h * scale_factor, tile_size_w, tile_size_h);
             }
             output = tiled_compute(
                 input,
@@ -290,8 +375,8 @@ struct VAE : public GGMLRunner {
                 static_cast<int>(W),
                 static_cast<int>(H),
                 scale_factor,
-                tile_size_x,
-                tile_size_y,
+                tile_size_w,
+                tile_size_h,
                 tile_overlap,
                 circular_x,
                 circular_y,
@@ -306,7 +391,9 @@ struct VAE : public GGMLRunner {
                                                   tiling_params);
         }
 
-        runner_end();
+        if (auto_end) {
+            runner_end();
+        }
 
         if (output.empty()) {
             LOG_ERROR("vae decode compute failed");
diff --git a/src/model/vae/wan_vae.hpp b/src/model/vae/wan_vae.hpp
index 035deae324..9a05839cfd 100644
--- a/src/model/vae/wan_vae.hpp
+++ b/src/model/vae/wan_vae.hpp
@@ -24,6 +24,7 @@ namespace WAN {
         std::tuple<int, int, int> padding;
         std::tuple<int, int, int> dilation;
         bool bias;
+        float scale = 1.f;
 
         void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override {
             auto weight = tensor_storage_map.find(prefix + "weight");
@@ -60,6 +61,10 @@ namespace WAN {
               dilation(std::move(dilation)),
               bias(bias) {}
 
+        void set_scale(float scale_value) {
+            scale = scale_value;
+        }
+
         ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x, ggml_tensor* cache_x = nullptr) {
             // x: [N*IC, ID, IH, IW]
             // result: x: [N*OC, ID, IH, IW]
@@ -82,11 +87,25 @@ namespace WAN {
             }
 
             x = ggml_ext_pad_ext(ctx->ggml_ctx, ctx->backend, x, lp0, rp0, lp1, rp1, lp2, rp2, 0, 0, ctx->circular_x_enabled, ctx->circular_y_enabled);
+            if (w->ne[2] == 1 && x->ne[2] == 1 && x->ne[3] == in_channels) {
+                // One frame through a one-frame-deep kernel is a 2D conv; backends without
+                // im2col_3d (Metal) otherwise fall back to a much slower direct conv_3d.
+                if (!ggml_is_contiguous(x)) {
+                    x = ggml_cont(ctx->ggml_ctx, x);
+                }
+                ggml_tensor* x2 = ggml_reshape_4d(ctx->ggml_ctx, x, x->ne[0], x->ne[1], in_channels, 1);
+                ggml_tensor* w2 = ggml_reshape_4d(ctx->ggml_ctx, w, w->ne[0], w->ne[1], in_channels, out_channels);
+                x2              = ggml_ext_conv_2d(ctx->ggml_ctx, x2, w2, b,
+                                                   std::get<2>(stride), std::get<1>(stride), 0, 0,
+                                                   std::get<2>(dilation), std::get<1>(dilation),
+                                                   ctx->conv2d_direct_enabled, false, false, scale);
+                return ggml_reshape_4d(ctx->ggml_ctx, x2, x2->ne[0], x2->ne[1], 1, out_channels);
+            }
             return ggml_ext_conv_3d(ctx->ggml_ctx, ctx->backend, x, w, b, in_channels,
                                     std::get<2>(stride), std::get<1>(stride), std::get<0>(stride),
                                     0, 0, 0,
                                     std::get<2>(dilation), std::get<1>(dilation), std::get<0>(dilation),
-                                    false, ctx->conv3d_direct_enabled);
+                                    false, ctx->conv3d_direct_enabled, scale);
         }
     };
 
@@ -128,14 +147,15 @@ namespace WAN {
         using Conv2d::Conv2d;
 
         ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            // With T == 1 or C == 1 this swap moves no data, so ggml_ext_cont keeps the view (no copy).
             ggml_tensor* x_swapped = ggml_permute(ctx->ggml_ctx, x, 0, 1, 3, 2);
-            x_swapped              = ggml_cont(ctx->ggml_ctx, x_swapped);
+            x_swapped              = ggml_ext_cont(ctx->ggml_ctx, x_swapped);
 
             ggml_tensor* out = Conv2d::forward(ctx, x_swapped);
 
             ggml_tensor* out_swapped = ggml_permute(ctx->ggml_ctx, out, 0, 1, 3, 2);
 
-            out_swapped = ggml_cont(ctx->ggml_ctx, out_swapped);
+            out_swapped = ggml_ext_cont(ctx->ggml_ctx, out_swapped);
 
             return out_swapped;
         }
@@ -1065,7 +1085,7 @@ namespace WAN {
 
                 _conv_num     = 34;
                 _enc_conv_num = 26;
-            } else if (version == VERSION_QWEN_IMAGE_LAYERED) {
+            } else if (version == VERSION_QWEN_IMAGE_LAYERED || version == VERSION_MING_IMAGE) {
                 input_channels = 4;
             }
 
@@ -1103,6 +1123,19 @@ namespace WAN {
             } else {
                 blocks["conv2"] = std::shared_ptr<GGMLBlock>(new CausalConv3d(z_dim, z_dim, {1, 1, 1}));
             }
+            if (version == VERSION_QWEN_IMAGE_2_1) {
+                // Keep large VAE activations within the FP16 convolution range.
+                const float conv_scale = 1.f / 128.f;
+                std::vector<GGMLBlock*> all_blocks;
+                get_all_blocks(all_blocks);
+                for (auto block : all_blocks) {
+                    if (auto conv = dynamic_cast<Conv2d*>(block)) {
+                        conv->set_scale(conv_scale);
+                    } else if (auto conv = dynamic_cast<CausalConv3d*>(block)) {
+                        conv->set_scale(conv_scale);
+                    }
+                }
+            }
         }
 
         static ggml_tensor* patchify(ggml_context* ctx,
@@ -1391,11 +1424,17 @@ namespace WAN {
         }
 
         sd::Tensor<float> diffusion_to_vae_latents(const sd::Tensor<float>& latents) override {
+            if (version == VERSION_MING_IMAGE) {
+                return latents / 8.0064f;
+            }
             auto [mean_tensor, std_tensor] = get_latents_mean_std(latents);
             return (latents * std_tensor) / scale_factor + mean_tensor;
         }
 
         sd::Tensor<float> vae_to_diffusion_latents(const sd::Tensor<float>& latents) override {
+            if (version == VERSION_MING_IMAGE) {
+                return latents * 8.0064f;
+            }
             auto [mean_tensor, std_tensor] = get_latents_mean_std(latents);
             return ((latents - mean_tensor) * scale_factor) / std_tensor;
         }
@@ -1429,15 +1468,14 @@ namespace WAN {
 
             ggml_tensor* z = make_input(z_tensor);
 
-            auto runner_ctx = get_context();
+            auto runner_ctx = get_context(gf);
 
             ggml_tensor* out = ae.decode_tiled_chunk(&runner_ctx, z, chunk_idx);
 
             for (size_t feat_idx = 0; feat_idx < ae._feat_map.size(); feat_idx++) {
                 ggml_tensor* feat_cache = ae._feat_map[feat_idx];
                 if (feat_cache != nullptr) {
-                    cache("feat_idx:" + std::to_string(feat_idx), feat_cache);
-                    ggml_build_forward_expand(gf, feat_cache);
+                    runner_ctx.persist_cache_tensor("feat_idx:" + std::to_string(feat_idx), feat_cache);
                 }
             }
 
diff --git a/src/model_io/gguf_io.cpp b/src/model_io/gguf_io.cpp
index 081fcf978f..8120131ea6 100644
--- a/src/model_io/gguf_io.cpp
+++ b/src/model_io/gguf_io.cpp
@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <cstdint>
 #include <cstdio>
+#include <filesystem>
 #include <fstream>
 #include <ostream>
 #include <string>
@@ -19,7 +20,7 @@ static void set_error(std::string* error, const std::string& message) {
 }
 
 bool is_gguf_file(const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -47,10 +48,14 @@ bool read_gguf_file(const std::string& file_path,
     gguf_context* ctx_gguf_ = nullptr;
     ggml_context* ctx_meta_ = nullptr;
 
-    ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_});
+    GGUFReader gguf_reader;
+    bool probe_ok = gguf_reader.load(file_path);
+
+    if (!probe_ok || !gguf_reader.has_tensors_beyond_ggml_limits()) {
+        ctx_gguf_ = gguf_init_from_file(file_path.c_str(), {true, &ctx_meta_});
+    }
     if (!ctx_gguf_) {
-        GGUFReader gguf_reader;
-        if (!gguf_reader.load(file_path)) {
+        if (!probe_ok && !gguf_reader.load(file_path)) {
             set_error(error, "failed to open '" + file_path + "' with GGUFReader");
             return false;
         }
@@ -174,7 +179,7 @@ bool GGUFStreamingWriter::write_metadata(const std::string& file_path,
     }
 
     LOG_INFO("trying to save tensors to %s", file_path.c_str());
-    FILE* file = fopen(file_path.c_str(), "wb+");
+    FILE* file = ggml_fopen(file_path.c_str(), "wb+");
     if (file == nullptr) {
         set_error(error, "failed to open output file '" + file_path + "'");
         close();
diff --git a/src/model_io/gguf_reader_ext.h b/src/model_io/gguf_reader_ext.h
index 7b372e4f59..f02fe3c6f4 100644
--- a/src/model_io/gguf_reader_ext.h
+++ b/src/model_io/gguf_reader_ext.h
@@ -2,6 +2,7 @@
 #define __SD_MODEL_IO_GGUF_READER_EXT_H__
 
 #include <cstdint>
+#include <filesystem>
 #include <fstream>
 #include <string>
 #include <vector>
@@ -35,23 +36,72 @@ enum class GGUFMetadataType : uint32_t {
 class GGUFReader {
 private:
     std::vector<GGUFTensorInfo> tensors_;
+    bool has_wide_tensors_    = false;
+    uint64_t remaining_bytes_ = 0;
     size_t data_offset_;
     size_t alignment_ = 32;  // default alignment is 32
 
     template <typename T>
     bool safe_read(std::ifstream& fin, T& value) {
-        fin.read(reinterpret_cast<char*>(&value), sizeof(T));
-        return fin.good();
+        return safe_read(fin, reinterpret_cast<char*>(&value), sizeof(T));
     }
 
     bool safe_read(std::ifstream& fin, char* buffer, size_t size) {
+        if (size > remaining_bytes_)
+            return false;
         fin.read(buffer, size);
-        return fin.good();
+        if (!fin.good())
+            return false;
+        remaining_bytes_ -= size;
+        return true;
+    }
+
+    bool safe_skip(std::ifstream& fin, uint64_t count, uint64_t element_size = 1) {
+        if (count > remaining_bytes_ / element_size)
+            return false;
+        uint64_t size = count * element_size;
+        fin.seekg(static_cast<std::streamoff>(size), std::ios::cur);
+        if (!fin.good())
+            return false;
+        remaining_bytes_ -= size;
+        return true;
     }
 
-    bool safe_seek(std::ifstream& fin, std::streamoff offset, std::ios::seekdir dir) {
-        fin.seekg(offset, dir);
-        return fin.good();
+    bool skip_metadata_values(std::ifstream& fin, GGUFMetadataType type, uint64_t count) {
+        switch (type) {
+            case GGUFMetadataType::UINT8:
+            case GGUFMetadataType::INT8:
+            case GGUFMetadataType::BOOL:
+                return safe_skip(fin, count);
+
+            case GGUFMetadataType::UINT16:
+            case GGUFMetadataType::INT16:
+                return safe_skip(fin, count, 2);
+
+            case GGUFMetadataType::UINT32:
+            case GGUFMetadataType::INT32:
+            case GGUFMetadataType::FLOAT32:
+                return safe_skip(fin, count, 4);
+
+            case GGUFMetadataType::UINT64:
+            case GGUFMetadataType::INT64:
+            case GGUFMetadataType::FLOAT64:
+                return safe_skip(fin, count, 8);
+
+            case GGUFMetadataType::STRING:
+                if (count > remaining_bytes_ / sizeof(uint64_t))
+                    return false;
+                for (uint64_t i = 0; i < count; i++) {
+                    uint64_t len = 0;
+                    if (!safe_read(fin, len) || !safe_skip(fin, len))
+                        return false;
+                }
+                return true;
+
+            default:
+                LOG_ERROR("Unknown metadata type=%u", static_cast<uint32_t>(type));
+                return false;
+        }
     }
 
     bool read_metadata(std::ifstream& fin) {
@@ -84,52 +134,12 @@ class GGUFReader {
             return true;
         }
 
-        switch (static_cast<GGUFMetadataType>(type)) {
-            case GGUFMetadataType::UINT8:
-            case GGUFMetadataType::INT8:
-            case GGUFMetadataType::BOOL:
-                return safe_seek(fin, 1, std::ios::cur);
-
-            case GGUFMetadataType::UINT16:
-            case GGUFMetadataType::INT16:
-                return safe_seek(fin, 2, std::ios::cur);
-
-            case GGUFMetadataType::UINT32:
-            case GGUFMetadataType::INT32:
-            case GGUFMetadataType::FLOAT32:
-                return safe_seek(fin, 4, std::ios::cur);
-
-            case GGUFMetadataType::UINT64:
-            case GGUFMetadataType::INT64:
-            case GGUFMetadataType::FLOAT64:
-                return safe_seek(fin, 8, std::ios::cur);
-
-            case GGUFMetadataType::STRING: {
-                uint64_t len = 0;
-                if (!safe_read(fin, len))
-                    return false;
-                return safe_seek(fin, len, std::ios::cur);
-            }
-
-            case GGUFMetadataType::ARRAY: {
-                uint32_t elem_type = 0;
-                uint64_t len       = 0;
-                if (!safe_read(fin, elem_type))
-                    return false;
-                if (!safe_read(fin, len))
-                    return false;
-
-                for (uint64_t i = 0; i < len; i++) {
-                    if (!read_metadata(fin))
-                        return false;
-                }
-                return true;
-            }
-
-            default:
-                LOG_ERROR("Unknown metadata type=%u", type);
+        uint64_t count = 1;
+        if (type == static_cast<uint32_t>(GGUFMetadataType::ARRAY)) {
+            if (!safe_read(fin, type) || !safe_read(fin, count))
                 return false;
         }
+        return skip_metadata_values(fin, static_cast<GGUFMetadataType>(type), count);
     }
 
     GGUFTensorInfo read_tensor_info(std::ifstream& fin) {
@@ -154,6 +164,7 @@ class GGUFReader {
         }
 
         if (n_dims > GGML_MAX_DIMS) {
+            has_wide_tensors_ = true;
             for (uint32_t i = GGML_MAX_DIMS; i < n_dims; i++) {
                 info.shape[GGML_MAX_DIMS - 1] *= info.shape[i];  // stack to last dim;
             }
@@ -174,12 +185,20 @@ class GGUFReader {
 
 public:
     bool load(const std::string& file_path) {
-        std::ifstream fin(file_path, std::ios::binary);
+        std::ifstream fin(std::filesystem::u8path(file_path), std::ios::binary | std::ios::ate);
         if (!fin) {
             LOG_ERROR("failed to open '%s'", file_path.c_str());
             return false;
         }
 
+        std::streamoff file_size = fin.tellg();
+        if (file_size < 0)
+            return false;
+        remaining_bytes_ = static_cast<uint64_t>(file_size);
+        fin.seekg(0, std::ios::beg);
+        if (!fin.good())
+            return false;
+
         // --- Header ---
         char magic[4];
         if (!safe_read(fin, magic, 4) || strncmp(magic, "GGUF", 4) != 0) {
@@ -228,6 +247,8 @@ class GGUFReader {
     }
 
     const std::vector<GGUFTensorInfo>& tensors() const { return tensors_; }
+
+    bool has_tensors_beyond_ggml_limits() const { return has_wide_tensors_; }
     size_t data_offset() const { return data_offset_; }
 };
 
diff --git a/src/model_io/safetensors_io.cpp b/src/model_io/safetensors_io.cpp
index 3e69742371..cf424af921 100644
--- a/src/model_io/safetensors_io.cpp
+++ b/src/model_io/safetensors_io.cpp
@@ -26,15 +26,15 @@ static void set_error(std::string* error, const std::string& message) {
 }
 
 static std::string resolve_index_shard_path(const std::string& index_path, const std::string& shard_path) {
-    fs::path shard_fs_path(shard_path);
+    fs::path shard_fs_path = fs::u8path(shard_path);
     if (shard_fs_path.is_absolute()) {
-        return shard_fs_path.lexically_normal().string();
+        return shard_fs_path.lexically_normal().u8string();
     }
-    return (fs::path(index_path).parent_path() / shard_fs_path).lexically_normal().string();
+    return (fs::u8path(index_path).parent_path() / shard_fs_path).lexically_normal().u8string();
 }
 
 bool is_safetensors_file(const std::string& file_path) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         return false;
     }
@@ -153,7 +153,7 @@ bool read_safetensors_file(const std::string& file_path,
                            std::string* error,
                            std::map<std::string, std::string>* metadata,
                            std::set<std::string>* tensor_names) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;
@@ -418,7 +418,7 @@ bool read_safetensors_index_file(const std::string& file_path,
                                  std::string* error) {
     shard_paths.clear();
 
-    std::ifstream file(file_path);
+    std::ifstream file(fs::u8path(file_path));
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;
@@ -522,7 +522,7 @@ bool write_safetensors_file(const std::string& file_path,
 
     const std::string header_str = header.dump();
 
-    std::ofstream file(file_path, std::ios::binary);
+    std::ofstream file(fs::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "' for writing");
         return false;
@@ -596,7 +596,7 @@ bool SafetensorsStreamingWriter::write_metadata(const std::string& file_path,
     data_start_                  = ST_HEADER_SIZE_LEN + header_str.size();
 
     LOG_INFO("trying to save tensors to %s", file_path.c_str());
-    std::ofstream file(file_path, std::ios::binary | std::ios::trunc);
+    std::ofstream file(fs::u8path(file_path), std::ios::binary | std::ios::trunc);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "' for writing");
         return false;
diff --git a/src/model_io/tensor_storage.h b/src/model_io/tensor_storage.h
index 1916d1f5e5..682a52217f 100644
--- a/src/model_io/tensor_storage.h
+++ b/src/model_io/tensor_storage.h
@@ -67,6 +67,18 @@ struct TensorStorage {
         }
     }
 
+    bool maps_directly_to(const ggml_tensor* dst) const {
+        if (is_f64 || is_i64 ||
+#ifdef SD_USE_UPSTREAM_GGML
+            is_f8_e4m3 || is_f8_e5m2 ||
+#endif
+            type != dst->type) {
+            return false;
+        }
+        return ne[0] == dst->ne[0] && ne[1] == dst->ne[1] && ne[2] == dst->ne[2] && ne[3] == dst->ne[3] &&
+               static_cast<size_t>(nbytes()) == ggml_nbytes(dst);
+    }
+
     void unsqueeze() {
         if (n_dims == 2) {
             n_dims = 4;
diff --git a/src/model_io/torch_legacy_io.cpp b/src/model_io/torch_legacy_io.cpp
index 3170fe8d4c..163e6649e3 100644
--- a/src/model_io/torch_legacy_io.cpp
+++ b/src/model_io/torch_legacy_io.cpp
@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <cstdint>
+#include <filesystem>
 #include <fstream>
 #include <string>
 #include <unordered_map>
@@ -98,7 +99,7 @@ static std::string torch_legacy_diagnostics(const std::string& file_path, const
 bool read_torch_legacy_file(const std::string& file_path,
                             std::vector<TensorStorage>& tensor_storages,
                             std::string* error) {
-    std::ifstream file(file_path, std::ios::binary);
+    std::ifstream file(std::filesystem::u8path(file_path), std::ios::binary);
     if (!file.is_open()) {
         set_error(error, "failed to open '" + file_path + "'");
         return false;
diff --git a/src/model_loader.cpp b/src/model_loader.cpp
index 57cc11ff28..90ab65eed5 100644
--- a/src/model_loader.cpp
+++ b/src/model_loader.cpp
@@ -4,6 +4,7 @@
 #include <cinttypes>
 #include <cstdarg>
 #include <cstdlib>
+#include <filesystem>
 #include <fstream>
 #include <functional>
 #include <mutex>
@@ -115,7 +116,7 @@ void convert_tensor(void* src,
                     ggml_type dst_type,
                     int nrows,
                     int n_per_row,
-                    std::vector<float> imatrix = {}) {
+                    std::vector<float> imatrix) {
     int n = nrows * n_per_row;
     if (src_type == dst_type) {
         size_t nbytes = n * ggml_type_size(src_type) / ggml_blck_size(src_type);
@@ -322,12 +323,18 @@ bool ModelLoader::init_from_safetensors_index_file(const std::string& file_path,
     }
 
     for (const std::string& shard_path : shard_paths) {
-        if (!parse_file(shard_path, prefix)) {
+        FileStamp stamp;
+        if (!read_file_stamp(shard_path, stamp)) {
+            return false;
+        }
+        parsed_dependencies_.push_back(stamp);
+        LOG_INFO("load %s using safetensors format", shard_path.c_str());
+        if (!init_from_safetensors_file(shard_path, prefix)) {
             return false;
         }
     }
 
-    std::ifstream index_file(file_path);
+    std::ifstream index_file(std::filesystem::u8path(file_path));
     const auto index = nlohmann::json::parse(index_file);
     for (const auto& entry : index.at("weight_map").items()) {
         const auto expected = (std::filesystem::u8path(file_path).parent_path() /
@@ -432,6 +439,16 @@ bool ModelLoader::init_from_diffusers_file(const std::string& file_path, const s
 SDVersion ModelLoader::get_sd_version() const {
     TensorStorage token_embedding_weight, input_block_weight, context_ebedding_weight;
 
+    if (tensor_storage_map.find("model.diffusion_model.emb_in.image.weight") != tensor_storage_map.end() &&
+        tensor_storage_map.find("model.diffusion_model.txt_mode_blocks.0.q_proj.weight") != tensor_storage_map.end()) {
+        return VERSION_FLUX3;
+    }
+
+    if (tensor_storage_map.count("model.diffusion_model.video_embedder.weight") &&
+        tensor_storage_map.count("model.diffusion_model.shared_block_modulation.weight")) {
+        return VERSION_MOSS_IMAGE;
+    }
+
     bool has_multiple_encoders = false;
     bool is_unet               = false;
 
@@ -518,6 +535,12 @@ SDVersion ModelLoader::get_sd_version() const {
             return VERSION_LLADA_IMAGE;
         }
         if (tensor_storage.name.find("model.diffusion_model.cap_embedder.0.weight") != std::string::npos) {
+            if (tensor_storage_map.find("text_encoders.llm.connector.layers.0.self_attn.q_proj.weight") != tensor_storage_map.end()) {
+                return VERSION_MING_IMAGE;
+            }
+            if (tensor_storage_map.find("model.diffusion_model.local_decoder.out_conv.weight") != tensor_storage_map.end()) {
+                return VERSION_Z_IMAGE_L2P;
+            }
             return VERSION_Z_IMAGE;
         }
         if (tensor_storage.name.find("double_stream_layers.0.img_instruct_attn.processor.img_to_q.weight") != std::string::npos) {
@@ -526,13 +549,28 @@ SDVersion ModelLoader::get_sd_version() const {
         if (tensor_storage.name.find("model.diffusion_model.layers.0.adaLN_sa_ln.weight") != std::string::npos) {
             return VERSION_ERNIE_IMAGE;
         }
+        if (tensor_storage.name.find("model.diffusion_model.t_block.1.weight") != std::string::npos &&
+            tensor_storage_map.find("model.diffusion_model.x_embedder.proj.weight") != tensor_storage_map.end() &&
+            tensor_storage_map.find("model.diffusion_model.audio_patchify_proj.weight") == tensor_storage_map.end()) {
+            return VERSION_PIXART;
+        }
         if (tensor_storage.name.find("model.diffusion_model.adaln_single.emb.timestep_embedder.linear_1.bias") != std::string::npos) {
+            // PixArt shares this timestep embedding with LTX-AV.
+            if (tensor_storage_map.find("model.diffusion_model.pos_embed.proj.weight") != tensor_storage_map.end() &&
+                tensor_storage_map.find("model.diffusion_model.adaln_single.linear.weight") != tensor_storage_map.end() &&
+                tensor_storage_map.find("model.diffusion_model.audio_patchify_proj.weight") == tensor_storage_map.end()) {
+                return VERSION_PIXART;
+            }
             return VERSION_LTXAV;
         }
         if (tensor_storage.name.find("model.diffusion_model.video_patch_proj.weight") != std::string::npos &&
             tensor_storage_map.find("model.diffusion_model.audio_patch_proj.weight") != tensor_storage_map.end()) {
             return VERSION_MINIMAX_H3;
         }
+        if (name == "model.diffusion_model.y_embedder.layer_pool.weight" &&
+            tensor_storage_map.count("model.diffusion_model.modulation_cores.adaln_img.weight")) {
+            return VERSION_IRIS;
+        }
         if (tensor_storage.name.find("model.diffusion_model.blocks.0.cross_attn.norm_k.weight") != std::string::npos) {
             is_wan = true;
         }
@@ -603,7 +641,7 @@ SDVersion ModelLoader::get_sd_version() const {
         }
     }
     if (is_wan) {
-        LOG_VERBOSE("patch_embedding_channels %d", patch_embedding_channels);
+        LOG_DEBUG("patch_embedding_channels %d", patch_embedding_channels);
         if (is_s2v) {
             return VERSION_WAN2_2_S2V;
         }
@@ -874,7 +912,8 @@ void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) {
 
 std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
                                                        std::set<std::string> ignore_tensors,
-                                                       bool writable_mmap) {
+                                                       bool writable_mmap,
+                                                       ggml_backend_dev_t device) {
     std::set<std::string> names;
     for (const auto& entry : tensors) {
         names.insert(entry.first);
@@ -896,6 +935,39 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
         if (!fdata.mmbuffer)
             continue;
 
+        // Wrapped on first use: a device buffer makes the whole file resident on that device.
+        std::shared_ptr<struct ggml_backend_buffer> file_buffer = device == nullptr ? fdata.mmbuffer : nullptr;
+        bool file_unmappable                                    = false;
+
+        auto buffer_for_file = [&]() -> ggml_backend_buffer_t {
+            if (file_buffer || file_unmappable) {
+                return file_buffer.get();
+            }
+            auto cached = fdata.device_mmbuffers.find(device);
+            if (cached != fdata.device_mmbuffers.end()) {
+                file_buffer = cached->second;
+                return file_buffer.get();
+            }
+            size_t max_tensor_size = 0;
+            for (const auto& ts : fdata.tensors) {
+                max_tensor_size = std::max(max_tensor_size, static_cast<size_t>(ts.nbytes()));
+            }
+            ggml_backend_buffer_t buf = sd_backend_dev_buffer_from_host_ptr(device,
+                                                                            fdata.mmapped->writable_data(),
+                                                                            fdata.mmapped->size(),
+                                                                            max_tensor_size);
+            if (buf == nullptr) {
+                LOG_WARN("mmap: %s cannot map '%s', loading it instead",
+                         ggml_backend_dev_name(device), fdata.path.c_str());
+                file_unmappable = true;
+                return nullptr;
+            }
+            LOG_INFO("mmap: mapped '%s' for %s", fdata.path.c_str(), ggml_backend_dev_name(device));
+            file_buffer                    = std::shared_ptr<struct ggml_backend_buffer>(buf, ggml_backend_buffer_free);
+            fdata.device_mmbuffers[device] = file_buffer;
+            return file_buffer.get();
+        };
+
         const std::vector<TensorStorage>& file_tensors = fdata.tensors;
 
         size_t file_mapped_bytes   = 0;
@@ -922,32 +994,21 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
             if (dst_tensor == nullptr)
                 continue;
 
-            if (tensor_storage.is_f64 ||
-                tensor_storage.is_i64 ||
-#ifdef SD_USE_UPSTREAM_GGML
-                tensor_storage.is_f8_e4m3 ||
-                tensor_storage.is_f8_e5m2 ||
-#endif
-                tensor_storage.type != dst_tensor->type) {
+            if (!tensor_storage.maps_directly_to(dst_tensor)) {
+                // let load_tensors worry about this
                 continue;
             }
 
             size_t tensor_size   = tensor_storage.nbytes();
             size_t tensor_offset = tensor_storage.offset;
 
-            if (tensor_storage.ne[0] != dst_tensor->ne[0] ||
-                tensor_storage.ne[1] != dst_tensor->ne[1] ||
-                tensor_storage.ne[2] != dst_tensor->ne[2] ||
-                tensor_storage.ne[3] != dst_tensor->ne[3] ||
-                tensor_size != ggml_nbytes(dst_tensor)) {
-                // let load_tensors worry about this
-                continue;
+            ggml_backend_buffer_t buf_mmap = buffer_for_file();
+            if (buf_mmap == nullptr) {
+                break;
             }
-
-            ggml_backend_buffer_t buf_mmap = fdata.mmbuffer.get();
-            uint8_t* mmap_data             = static_cast<uint8_t*>(ggml_backend_buffer_get_base(buf_mmap));
-            dst_tensor->buffer             = buf_mmap;
-            dst_tensor->data               = mmap_data + tensor_offset;
+            uint8_t* mmap_data = static_cast<uint8_t*>(ggml_backend_buffer_get_base(buf_mmap));
+            dst_tensor->buffer = buf_mmap;
+            dst_tensor->data   = mmap_data + tensor_offset;
 
             file_mapped_bytes += tensor_size;
             file_mapped_tensors++;
@@ -956,7 +1017,7 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
         if (file_mapped_bytes > 0) {
             mapped_tensors += file_mapped_tensors;
             mapped_bytes += file_mapped_bytes;
-            result.push_back({fdata.mmapped, fdata.mmbuffer});
+            result.push_back({fdata.mmapped, file_buffer});
         }
     }
 
@@ -972,6 +1033,16 @@ std::vector<MmapTensorStore> ModelLoader::mmap_tensors(std::map<std::string, ggm
     return result;
 }
 
+std::vector<ggml_backend_buffer_t> ModelLoader::get_device_mmap_buffers() const {
+    std::vector<ggml_backend_buffer_t> buffers;
+    for (const auto& fdata : file_data) {
+        for (const auto& entry : fdata.device_mmbuffers) {
+            buffers.push_back(entry.second.get());
+        }
+    }
+    return buffers;
+}
+
 bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                                bool enable_mmap,
                                const std::set<std::string>* target_tensor_names,
@@ -1037,10 +1108,12 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
         if (tensors_to_process.empty()) {
             continue;
         }
-        LOG_VERBOSE("loading %zu/%zu tensors from %s",
-                    tensors_to_process.size(),
-                    file_tensors.size(),
-                    file_path.c_str());
+        if (log_progress) {
+            LOG_VERBOSE("loading %zu/%zu tensors from %s",
+                        tensors_to_process.size(),
+                        file_tensors.size(),
+                        file_path.c_str());
+        }
 
         bool is_zip = fdata.is_zip;
 
@@ -1069,7 +1142,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                         return;
                     }
                 } else if (!mmapped) {
-                    file.open(file_path, std::ios::binary);
+                    file.open(std::filesystem::u8path(file_path), std::ios::binary);
                     if (!file.is_open()) {
                         LOG_ERROR("failed to open '%s'", file_path.c_str());
                         failed = true;
@@ -1115,6 +1188,11 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                     if (dst_tensor->buffer != nullptr && dst_tensor->buffer == fdata.mmbuffer.get()) {
                         continue;
                     }
+                    if (dst_tensor->buffer != nullptr &&
+                        std::any_of(fdata.device_mmbuffers.begin(), fdata.device_mmbuffers.end(),
+                                    [&](const auto& entry) { return entry.second.get() == dst_tensor->buffer; })) {
+                        continue;
+                    }
 
                     size_t nbytes_to_read = tensor_storage.nbytes_to_read();
 
@@ -1551,6 +1629,9 @@ bool ModelLoader::tensor_should_be_converted(const TensorStorage& tensor_storage
             // Pass, do not convert. For Unet
         } else if (contains(name, "embedding")) {
             // Pass, do not convert embedding
+        } else if (contains(name, "scale_shift_table")) {
+            // Pass, do not convert. adaLN modulation tables (PixArt, LTXV) are sliced
+            // element-wise, which is invalid on quantized block layouts.
         } else if (ends_with(name, "_pad_token")) {
             // Pass, do not convert. LLaDA-Image stores its pad tokens far outside the f16
             // range, so any format with an f16 scale or payload turns them into inf.
diff --git a/src/model_loader.h b/src/model_loader.h
index c5335b53a9..aae359099e 100644
--- a/src/model_loader.h
+++ b/src/model_loader.h
@@ -13,6 +13,14 @@
 
 TensorTypeRules parse_tensor_type_rules(const std::string& tensor_type_rules);
 
+void convert_tensor(void* src,
+                    ggml_type src_type,
+                    void* dst,
+                    ggml_type dst_type,
+                    int nrows,
+                    int n_per_row,
+                    std::vector<float> imatrix = {});
+
 class MmapWrapper;
 
 struct ModelFileData {
@@ -20,6 +28,8 @@ struct ModelFileData {
     std::vector<TensorStorage> tensors;
     std::shared_ptr<MmapWrapper> mmapped;
     std::shared_ptr<struct ggml_backend_buffer> mmbuffer;
+    // mmapped wrapped by devices that can use host memory in place (buffer_from_host_ptr)
+    std::map<ggml_backend_dev_t, std::shared_ptr<struct ggml_backend_buffer>> device_mmbuffers;
     bool is_zip;
 };
 
@@ -95,6 +105,8 @@ class ModelLoader {
     bool del_file(FileId id);
     uint64_t file_revision(FileId id) const;
     std::string file_path(FileId id) const;
+    std::map<std::string, std::string> file_metadata(FileId id) const;
+    String2TensorStorage file_tensors(FileId id) const;
     String2TensorStorage file_tensors(FileId id, SDVersion version) const;
     bool load_file_tensors(FileId id, SDVersion version, on_new_tensor_cb_t callback, const std::set<std::string>& names, bool use_mmap = false) const;
     bool refresh_files(bool include_isolated = true);
@@ -120,7 +132,9 @@ class ModelLoader {
     void process_model_files(bool enable_mmap = false, bool writable_mmap = true);
     std::vector<MmapTensorStore> mmap_tensors(std::map<std::string, ggml_tensor*>& tensors,
                                               std::set<std::string> ignore_tensors = {},
-                                              bool writable                        = true);
+                                              bool writable                        = true,
+                                              ggml_backend_dev_t device            = nullptr);
+    std::vector<ggml_backend_buffer_t> get_device_mmap_buffers() const;
     bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb,
                       bool use_mmap                                    = false,
                       const std::set<std::string>* target_tensor_names = nullptr,
diff --git a/src/model_loader_files.cpp b/src/model_loader_files.cpp
index 8e838b1190..9ac5456cc1 100644
--- a/src/model_loader_files.cpp
+++ b/src/model_loader_files.cpp
@@ -324,6 +324,24 @@ ModelLoader ModelLoader::file_reader(FileId id, SDVersion version) const {
     return reader;
 }
 
+std::map<std::string, std::string> ModelLoader::file_metadata(FileId id) const {
+    for (const auto& file : files_) {
+        if (file.id == id) {
+            return file.metadata;
+        }
+    }
+    return {};
+}
+
+String2TensorStorage ModelLoader::file_tensors(FileId id) const {
+    for (const auto& file : files_) {
+        if (file.id == id) {
+            return file.tensors;
+        }
+    }
+    return {};
+}
+
 String2TensorStorage ModelLoader::file_tensors(FileId id, SDVersion version) const {
     return file_reader(id, version).tensor_storage_map;
 }
diff --git a/src/model_manager.cpp b/src/model_manager.cpp
index 9a9fe06287..05ee55f676 100644
--- a/src/model_manager.cpp
+++ b/src/model_manager.cpp
@@ -4,6 +4,7 @@
 #include <cstdint>
 #include <iterator>
 #include <mutex>
+#include <set>
 #include <tuple>
 #include <unordered_set>
 
@@ -79,6 +80,10 @@ static bool device_supports_param_op(ggml_backend_dev_t device,
     if (op == GGML_OP_GET_ROWS) {
         ggml_tensor* indices = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1);
         op_tensor            = ggml_get_rows(ctx, weight, indices);
+    } else if (op == GGML_OP_CUSTOM) {
+        op_tensor         = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1);
+        op_tensor->op     = op;
+        op_tensor->src[0] = weight;
     }
     if (op_tensor == nullptr) {
         ggml_free(ctx);
@@ -246,20 +251,17 @@ bool ModelManager::register_param_tensors(ModelComponent component,
         }
         ggml_set_name(tensor, name.c_str());
 
-        auto state            = std::make_unique<TensorState>();
-        state->name           = name;
-        state->tensor         = tensor;
-        state->component      = component;
-        state->source_file    = source_file;
-        state->source_version = source_version;
-        auto source           = sources.find(name);
-        if (source != sources.end()) {
-            state->source     = source->second;
-            state->has_source = true;
-        }
+        auto state             = std::make_unique<TensorState>();
+        state->name            = name;
+        state->tensor          = tensor;
+        state->component       = component;
+        state->source_file     = source_file;
+        state->source_version  = source_version;
+        state->sources         = find_tensor_sources(*state, sources);
         state->residency_mode  = residency_mode;
         state->compute_backend = compute_backend;
         state->params_backend  = params_backend;
+        state->matmul_only     = sd_is_matmul_only_weight(tensor);
         auto split_buffer      = split_buffer_types_.find(compute_backend);
         if (allow_split_buffer && split_buffer != split_buffer_types_.end()) {
             state->split_buffer_type = split_buffer->second;
@@ -271,6 +273,7 @@ bool ModelManager::register_param_tensors(ModelComponent component,
                 state->usage_op = op_it->second;
             }
         }
+        sync_cpu_repack_tag(*state);
         new_states.push_back(std::move(state));
     }
 
@@ -511,7 +514,9 @@ bool ModelManager::stage_tensors_to_compute_backend(const std::vector<TensorStat
             LOG_ERROR("model manager params backend is null for tensor '%s'", state->name.c_str());
             return false;
         }
-        if (state->compute_backend == state->params_backend || state->staged_to_compute_backend) {
+        // Custom CPU operators must retain host weights even when the runner uses a GPU.
+        if (state->usage_op == GGML_OP_CUSTOM ||
+            state->compute_backend == state->params_backend || state->staged_to_compute_backend) {
             continue;
         }
         if (!state->loaded_to_params_backend || state->tensor == nullptr || state->tensor->data == nullptr) {
@@ -757,12 +762,33 @@ bool ModelManager::validate_tensor(const TensorState& state) const {
         return true;
     }
 
-    if (!state.has_source) {
+    if (state.sources.empty()) {
         LOG_ERROR("%s tensor '%s' not in model metadata", model_component_name(state.component), state.name.c_str());
         return false;
     }
 
-    const TensorStorage& tensor_storage = state.source;
+    TensorStorage tensor_storage = state.sources.front();
+    if (state.sources.size() > 1) {
+        const int dim = tensor_storage.n_dims - 1;
+        if (dim < 0 || dim >= GGML_MAX_DIMS) {
+            return false;
+        }
+        tensor_storage.ne[dim] = 0;
+        for (const auto& part : state.sources) {
+            if (part.n_dims != tensor_storage.n_dims || part.ne[dim] < 0 ||
+                part.ne[dim] > state.tensor->ne[dim] - tensor_storage.ne[dim]) {
+                LOG_ERROR("invalid tensor part '%s' for '%s'", part.name.c_str(), state.name.c_str());
+                return false;
+            }
+            for (int i = 0; i < GGML_MAX_DIMS; ++i) {
+                if (i != dim && part.ne[i] != tensor_storage.ne[i]) {
+                    LOG_ERROR("incompatible tensor part '%s' for '%s'", part.name.c_str(), state.name.c_str());
+                    return false;
+                }
+            }
+            tensor_storage.ne[dim] += part.ne[dim];
+        }
+    }
     if (state.tensor->ne[0] != tensor_storage.ne[0] ||
         state.tensor->ne[1] != tensor_storage.ne[1] ||
         state.tensor->ne[2] != tensor_storage.ne[2] ||
@@ -780,44 +806,93 @@ bool ModelManager::validate_tensor(const TensorState& state) const {
 
 bool ModelManager::mmap_params(const std::vector<TensorState*>& states,
                                std::vector<ParamsStorageBlock*>& created_storage_blocks) {
-    std::map<std::string, ggml_tensor*> mmap_candidates;
-    std::map<std::string, TensorState*> mmap_states;
+    // A GPU that computes on mmapped params in place cannot address a CPU buffer, and nothing
+    // stages them for it, so they are mapped through a buffer of that GPU's device.
+    struct MmapGroup {
+        std::map<std::string, ggml_tensor*> candidates;
+        std::map<std::string, TensorState*> states;
+    };
+    std::map<ggml_backend_dev_t, MmapGroup> groups;
     for (TensorState* state : states) {
         if (state == nullptr || !can_mmap_storage(*state) || state->tensor == nullptr ||
             state->tensor->data != nullptr || state->tensor->view_src != nullptr) {
             continue;
         }
-        mmap_candidates[state->name] = state->tensor;
-        mmap_states[state->name]     = state;
+        ggml_backend_dev_t device = nullptr;
+        if (!sd_backend_is_cpu(state->compute_backend) && !sd_backend_is_cpu(state->params_backend)) {
+            device = ggml_backend_get_device(state->compute_backend);
+        }
+        MmapGroup& group              = groups[device];
+        group.candidates[state->name] = state->tensor;
+        group.states[state->name]     = state;
     }
-    if (mmap_candidates.empty()) {
-        return true;
+
+    for (auto& [device, group] : groups) {
+        // Device buffers wrap read-only mappings only; params that LoRAs are merged into in place
+        // are loaded instead.
+        if (device != nullptr && writable_mmap_) {
+            continue;
+        }
+        auto mmap_store = model_loader_.mmap_tensors(group.candidates, {}, writable_mmap_, device);
+        if (mmap_store.empty()) {
+            continue;
+        }
+
+        auto block                = std::make_unique<ParamsStorageBlock>();
+        block->mmap_tensor_stores = std::move(mmap_store);
+        ParamsStorageBlock* raw   = block.get();
+        for (const auto& pair : group.states) {
+            TensorState* state = pair.second;
+            if (state != nullptr && state->tensor != nullptr && state->tensor->data != nullptr) {
+                block->states.push_back(state);
+            }
+        }
+
+        if (!block->states.empty()) {
+            params_storage_blocks_.push_back(std::move(block));
+            created_storage_blocks.push_back(raw);
+        }
     }
+    return true;
+}
 
-    auto mmap_store = model_loader_.mmap_tensors(mmap_candidates, {}, writable_mmap_);
-    if (mmap_store.empty()) {
-        return true;
+// The matmul-only tag both selects the repack buffer and sends a not yet allocated weight down the repacked-CPU
+// graph path, so keep it only on weights this context will repack. An allocated weight keeps its tag: its buffer decides.
+void ModelManager::sync_cpu_repack_tag(TensorState& state) const {
+    if (!state.matmul_only || state.tensor->buffer != nullptr) {
+        return;
+    }
+    sd_tag_matmul_only_weight(state.tensor);
+    bool repack = cpu_repack_allowed_ && state.usage_op == GGML_OP_NONE && state.params_backend == state.compute_backend &&
+                  sd_backend_is_cpu(state.compute_backend) &&
+                  sd_cpu_repack_buffer_type_for(ggml_backend_get_device(state.compute_backend), state.tensor) != nullptr;
+    if (repack && can_mmap_storage(state) && state.sources.size() == 1 && state.sources[0].maps_directly_to(state.tensor)) {
+        // Mapped weights cannot be repacked: log once.
+        static std::once_flag logged;
+        std::call_once(logged, [] { LOG_INFO("cpu weight repack: skipped for mmapped weights (drop --mmap to repack)"); });
+        repack = false;
     }
+    if (!repack) {
+        sd_untag_matmul_only_weight(state.tensor);
+    }
+}
 
-    auto block                = std::make_unique<ParamsStorageBlock>();
-    block->mmap_tensor_stores = std::move(mmap_store);
-    ParamsStorageBlock* raw   = block.get();
-    for (const auto& pair : mmap_states) {
-        TensorState* state = pair.second;
-        if (state != nullptr && state->tensor != nullptr && state->tensor->data != nullptr) {
-            block->states.push_back(state);
-        }
+void ModelManager::set_enable_mmap(bool enable_mmap) {
+    enable_mmap_ = enable_mmap;
+    for (auto& state : tensor_states_) {
+        sync_cpu_repack_tag(*state);
     }
+}
 
-    if (!block->states.empty()) {
-        params_storage_blocks_.push_back(std::move(block));
-        created_storage_blocks.push_back(raw);
+void ModelManager::set_cpu_repack_allowed(bool allowed) {
+    cpu_repack_allowed_ = allowed;
+    for (auto& state : tensor_states_) {
+        sync_cpu_repack_tag(*state);
     }
-    return true;
 }
 
 bool ModelManager::can_mmap_storage(const TensorState& state) const {
-    if (state.source_file != 0 || !enable_mmap_ || state.residency_mode != ResidencyMode::ParamBackend) {
+    if (state.sources.size() > 1 || state.source_file != 0 || !enable_mmap_ || state.residency_mode != ResidencyMode::ParamBackend) {
         return false;
     }
     if (state.compute_backend == nullptr || state.params_backend == nullptr) {
@@ -927,24 +1002,123 @@ bool ModelManager::alloc_params_buffers(const std::vector<TensorState*>& states,
     return true;
 }
 
+bool ModelManager::load_tensor_parts(TensorState& state) {
+    auto ctx = std::unique_ptr<ggml_context, decltype(&ggml_free)>(
+        ggml_init({state.sources.size() * ggml_tensor_overhead(), nullptr, true}), ggml_free);
+    if (!ctx) {
+        return false;
+    }
+    const size_t size = ggml_nbytes(state.tensor);
+    std::vector<uint8_t> buffer;
+    void* data = state.tensor->data;
+    if (!ggml_backend_buffer_is_host(state.tensor->buffer)) {
+        buffer.resize(size);
+        data = buffer.data();
+    }
+    std::map<std::string, ggml_tensor*> parts;
+    std::set<std::string> names;
+    size_t offset = 0;
+    for (const auto& source : state.sources) {
+        auto part              = ggml_new_tensor(ctx.get(), state.tensor->type, source.n_dims, source.ne);
+        const size_t part_size = ggml_nbytes(part);
+        if (part_size > size - offset) {
+            return false;
+        }
+        part->data         = static_cast<uint8_t*>(data) + offset;
+        parts[source.name] = part;
+        names.insert(source.name);
+        offset += part_size;
+    }
+    if (offset != size) {
+        return false;
+    }
+    std::set<std::string> loaded;
+    std::mutex mutex;
+    auto callback = [&](const TensorStorage& source, ggml_tensor** dst) {
+        *dst      = nullptr;
+        auto part = parts.find(source.name);
+        if (part != parts.end()) {
+            *dst = part->second;
+            std::lock_guard<std::mutex> lock(mutex);
+            loaded.insert(source.name);
+        }
+        return true;
+    };
+    const bool success = state.source_file == 0
+                             ? model_loader_.load_tensors(callback, enable_mmap_, &names, false)
+                             : model_loader_.load_file_tensors(state.source_file, state.source_version, callback, names, enable_mmap_);
+    if (!success || loaded != names) {
+        return false;
+    }
+    if (!buffer.empty()) {
+        // Upload the assembled tensor once, including for row-split backend buffers.
+        ggml_backend_tensor_set(state.tensor, buffer.data(), 0, size);
+    }
+    state.loaded_to_params_backend = true;
+    return true;
+}
+
 bool ModelManager::load_tensors(const std::vector<TensorState*>& states) {
     using ReadGroup = std::pair<ModelLoader::FileId, SDVersion>;
-    using ReadBatch = std::map<std::string, std::vector<TensorState*>>;
+    using ReadBatch = std::map<std::string, std::vector<ggml_tensor*>>;
     std::map<ReadGroup, std::vector<ReadBatch>> groups;
-    for (auto* state : states) {
-        if (state == nullptr)
-            continue;
-        auto& batches = groups[{state->source_file, state->source_version}];
-        // The loader supplies one destination per name; only conflicting types need another batch.
+    size_t part_count = 0;
+    for (const auto* state : states) {
+        if (state != nullptr && state->sources.size() > 1 && ggml_backend_buffer_is_host(state->tensor->buffer)) {
+            part_count += state->sources.size();
+        }
+    }
+    auto parts_ctx = std::unique_ptr<ggml_context, decltype(&ggml_free)>(
+        part_count > 0 ? ggml_init({part_count * ggml_tensor_overhead(), nullptr, true}) : nullptr, ggml_free);
+    if (part_count > 0 && !parts_ctx) {
+        return false;
+    }
+    auto add_destination = [&](const TensorState& state, const std::string& name, ggml_tensor* tensor) {
+        auto& batches = groups[{state.source_file, state.source_version}];
+        // The loader supplies one destination per name; incompatible destinations need another batch.
         auto batch = std::find_if(batches.begin(), batches.end(), [&](const ReadBatch& candidate) {
-            auto found = candidate.find(state->name);
-            return found == candidate.end() || found->second.front()->tensor->type == state->tensor->type;
+            auto found = candidate.find(name);
+            if (found == candidate.end()) {
+                return true;
+            }
+            // Others copy from the first destination, so a repacked first destination needs another batch.
+            const ggml_tensor* first = found->second.front();
+            return first->type == tensor->type && ggml_are_same_shape(first, tensor) &&
+                   !sd_weight_may_be_cpu_repacked(first);
         });
         if (batch == batches.end()) {
             batches.emplace_back();
             batch = std::prev(batches.end());
         }
-        (*batch)[state->name].push_back(state);
+        (*batch)[name].push_back(tensor);
+    };
+    for (auto* state : states) {
+        if (state == nullptr)
+            continue;
+        if (state->sources.size() > 1) {
+            if (ggml_backend_buffer_is_host(state->tensor->buffer)) {
+                const size_t size = ggml_nbytes(state->tensor);
+                size_t offset     = 0;
+                for (const auto& source : state->sources) {
+                    auto* part             = ggml_new_tensor(parts_ctx.get(), state->tensor->type, source.n_dims, source.ne);
+                    const size_t part_size = ggml_nbytes(part);
+                    if (part_size > size - offset) {
+                        return false;
+                    }
+                    part->data   = static_cast<uint8_t*>(state->tensor->data) + offset;
+                    part->buffer = state->tensor->buffer;
+                    add_destination(*state, source.name, part);
+                    offset += part_size;
+                }
+                if (offset != size) {
+                    return false;
+                }
+            } else if (!load_tensor_parts(*state)) {
+                return false;
+            }
+            continue;
+        }
+        add_destination(*state, state->name, state->tensor);
     }
     for (auto& group : groups) {
         for (auto& batch : group.second) {
@@ -958,7 +1132,7 @@ bool ModelManager::load_tensors(const std::vector<TensorState*>& states) {
                 auto found = batch.find(source.name);
                 if (found == batch.end())
                     return true;
-                *dst = found->second.front()->tensor;
+                *dst = found->second.front();
                 std::lock_guard<std::mutex> lock(mutex);
                 loaded.insert(source.name);
                 return true;
@@ -969,15 +1143,18 @@ bool ModelManager::load_tensors(const std::vector<TensorState*>& states) {
             if (!success || loaded != names)
                 return false;
             for (auto& entry : batch) {
-                auto* first = entry.second.front()->tensor;
-                for (auto* state : entry.second) {
-                    if (state->tensor != first)
-                        ggml_backend_tensor_copy(first, state->tensor);
-                    state->loaded_to_params_backend = true;
+                auto* first = entry.second.front();
+                for (auto* tensor : entry.second) {
+                    if (tensor != first)
+                        ggml_backend_tensor_copy(first, tensor);
                 }
             }
         }
     }
+    for (auto* state : states) {
+        if (state != nullptr)
+            state->loaded_to_params_backend = true;
+    }
     return true;
 }
 
@@ -998,6 +1175,20 @@ ggml_backend_buffer_type_t ModelManager::params_buffer_type_for(const TensorStat
     if (params_buft == nullptr) {
         params_buft = ggml_backend_get_default_buffer_type(state.params_backend);
     }
+    if (cpu_repack_allowed_ && state.usage_op == GGML_OP_NONE && state.params_backend == state.compute_backend &&
+        sd_backend_is_cpu(state.compute_backend)) {
+        ggml_backend_buffer_type_t repack_buft = sd_cpu_repack_buffer_type_for(ggml_backend_get_device(state.compute_backend), state.tensor);
+        if (repack_buft != nullptr) {
+            static std::mutex logged_mutex;
+            static std::set<std::string> logged;
+            std::lock_guard<std::mutex> lock(logged_mutex);
+            if (logged.insert(ggml_backend_buft_name(repack_buft)).second) {
+                LOG_INFO("cpu weight repack: quantized matmul weights use %s (SD_CPU_REPACK=0 to disable)",
+                         ggml_backend_buft_name(repack_buft));
+            }
+            return repack_buft;
+        }
+    }
     if (state.usage_op != GGML_OP_NONE &&
         state.compute_backend != nullptr) {
         ggml_backend_dev_t compute_dev = ggml_backend_get_device(state.compute_backend);
@@ -1296,6 +1487,7 @@ bool ModelManager::assign_compute_backend(const std::vector<ggml_tensor*>& tenso
         if (params_follow_compute) {
             state->params_backend = compute_backend;
         }
+        sync_cpu_repack_tag(*state);
     }
 
     return true;
@@ -1309,6 +1501,9 @@ size_t ModelManager::compute_backend_alloc_size(const std::vector<TensorState*>&
         if (state == nullptr || state->tensor == nullptr) {
             continue;
         }
+        if (state->usage_op == GGML_OP_CUSTOM && !sd_backend_is_cpu(state->compute_backend)) {
+            continue;
+        }
         const bool compute_resident =
             state->compute_backend == state->params_backend
                 ? state->loaded_to_params_backend
@@ -1353,15 +1548,16 @@ size_t ModelManager::compute_backend_resident_bytes(ggml_backend_t compute_backe
     }
 
     size_t total_size = 0;
-    auto add_buffer   = [&](ggml_backend_buffer_t buffer) {
-        if (buffer == nullptr || ggml_backend_buffer_is_host(buffer)) {
+    std::unordered_set<ggml_backend_buffer_t> seen;
+    auto add_buffer = [&](ggml_backend_buffer_t buffer) {
+        if (buffer == nullptr || ggml_backend_buffer_is_host(buffer) || !seen.insert(buffer).second) {
             return;
         }
         ggml_backend_buffer_type_t buffer_type = ggml_backend_buffer_get_type(buffer);
         auto split_devices                     = split_buffer_devices_.find(buffer_type);
         const bool on_device                   = split_devices == split_buffer_devices_.end()
-                                                       ? buffer_type != nullptr && ggml_backend_buft_get_device(buffer_type) == compute_device
-                                                       : std::any_of(split_devices->second.begin(), split_devices->second.end(), [&](const auto& entry) {
+                                                     ? buffer_type != nullptr && ggml_backend_buft_get_device(buffer_type) == compute_device
+                                                     : std::any_of(split_devices->second.begin(), split_devices->second.end(), [&](const auto& entry) {
                                          return ggml_backend_get_device(entry.first) == compute_device;
                                      });
         if (!on_device) {
@@ -1371,9 +1567,16 @@ size_t ModelManager::compute_backend_resident_bytes(ggml_backend_t compute_backe
         total_size               = buffer_size > SIZE_MAX - total_size ? SIZE_MAX : total_size + buffer_size;
     };
 
+    // The loader may retain device mappings after their parameter blocks are released.
+    for (ggml_backend_buffer_t buffer : model_loader_.get_device_mmap_buffers()) {
+        add_buffer(buffer);
+    }
     for (const auto& block : params_storage_blocks_) {
         if (block != nullptr) {
             add_buffer(block->buffer);
+            for (const auto& store : block->mmap_tensor_stores) {
+                add_buffer(store.mmbuffer.get());
+            }
         }
     }
     for (const auto& block : compute_staging_blocks_) {
@@ -1602,6 +1805,9 @@ void ModelManager::set_workspace_reclaimer(uintptr_t owner_id, std::function<boo
 
 void ModelManager::remove_runtime_owner(uintptr_t owner_id) {
     workspace_reclaimers_.erase(owner_id);
+    for (auto it = device_query_cache_.begin(); it != device_query_cache_.end();) {
+        it = it->first.first == owner_id ? device_query_cache_.erase(it) : std::next(it);
+    }
     for (auto it = runtime_residencies_.begin(); it != runtime_residencies_.end();) {
         if (it->first.first == owner_id) {
             it = runtime_residencies_.erase(it);
@@ -1613,7 +1819,8 @@ void ModelManager::remove_runtime_owner(uintptr_t owner_id) {
 
 ModelManager::CapacityCheck ModelManager::check_capacity(
     const DeviceMemoryRequest& request,
-    const std::vector<TensorState*>& states) const {
+    const std::vector<TensorState*>& states,
+    bool log_details) const {
     CapacityCheck result;
     if (request.compute_backend == nullptr || sd_backend_is_cpu(request.compute_backend)) {
         return result;
@@ -1630,23 +1837,64 @@ ModelManager::CapacityCheck ModelManager::check_capacity(
             return SIZE_MAX;
         }
         size_t free_bytes = 0, total_bytes = 0;
-        ggml_backend_dev_memory(device, &free_bytes, &total_bytes);
+        // only checks that allocate nothing reuse a reading, so it never predates this owner's growth
+        const bool reusable  = request.reuse_device_query && request.pending_allocation_bytes == 0 && missing == 0;
+        const auto cache_key = std::make_pair(request.owner_id, device);
+        auto cached          = device_query_cache_.find(cache_key);
+        if (reusable && cached != device_query_cache_.end()) {
+            free_bytes  = cached->second.first;
+            total_bytes = cached->second.second;
+        } else {
+            ggml_backend_dev_memory(device, &free_bytes, &total_bytes);
+            if (reusable) {
+                device_query_cache_[cache_key] = {free_bytes, total_bytes};
+            } else if (cached != device_query_cache_.end()) {
+                device_query_cache_.erase(cached);
+            }
+        }
+        const size_t weights_resident = compute_backend_resident_bytes(backend);
+        const size_t other_runtime    = other_runtime_resident_bytes(request.owner_id, backend);
+        const size_t resident         = add(weights_resident, add(other_runtime, request.runtime_resident_bytes));
+        if (log_details) {
+            LOG_WARN("model manager memory on %s: reported free %.2f MB / total %.2f MB, tracked weights %.2f MB / other runtime %.2f MB / current runtime %.2f MB",
+                        ggml_backend_name(backend),
+                        free_bytes / (1024.0 * 1024.0), total_bytes / (1024.0 * 1024.0),
+                        weights_resident / (1024.0 * 1024.0), other_runtime / (1024.0 * 1024.0),
+                        request.runtime_resident_bytes / (1024.0 * 1024.0));
+        }
         if (free_bytes == 0 && total_bytes == 0) {
             return SIZE_MAX;
         }
         // Vulkan's heap budget subtraction can underflow when usage exceeds the budget.
-        if (total_bytes > 0 && free_bytes > total_bytes) {
-            return size_t{0};
+        if (total_bytes > 0 && free_bytes > total_bytes && sd_backend_is(backend, "Vulkan")) {
+            if (ggml_backend_dev_type(device) != GGML_BACKEND_DEVICE_TYPE_IGPU) {
+                return size_t{0};
+            }
+            // A wrapped report may reflect real pressure; reclaim before accepting the tracked estimate.
+            LOG_VERBOSE("ignoring impossible free memory report on %s (%.2f MB free / %.2f MB total), using tracked residency",
+                           ggml_backend_name(backend), free_bytes / (1024.0 * 1024.0), total_bytes / (1024.0 * 1024.0));
+            free_bytes                  = total_bytes;
+            result.requires_reclamation = true;
         }
-        const size_t resident = add(compute_backend_resident_bytes(backend),
-                                       add(other_runtime_resident_bytes(request.owner_id, backend),
-                                           request.runtime_resident_bytes));
         if (total_bytes > 0) {
             free_bytes = std::min(free_bytes, resident < total_bytes ? total_bytes - resident : 0);
         }
         return free_bytes;
     };
     result.available_device_bytes = available_device_bytes(request.compute_backend);
+    struct DropReadingIfNoFit {
+        const CapacityCheck& result;
+        std::map<std::pair<uintptr_t, ggml_backend_dev_t>, std::pair<size_t, size_t>>& cache;
+        uintptr_t owner;
+        ~DropReadingIfNoFit() {
+            // A failed check leads to reclaiming / evicting; the retries must see fresh readings.
+            if (!result.fits()) {
+                for (auto it = cache.begin(); it != cache.end();) {
+                    it = it->first.first == owner ? cache.erase(it) : std::next(it);
+                }
+            }
+        }
+    } drop_reading_if_no_fit{result, device_query_cache_, request.owner_id};
     if (request.max_backend_bytes > 0) {
         const size_t resident         = add(compute_backend_resident_bytes(request.compute_backend),
                                             other_runtime_resident_bytes(request.owner_id, request.compute_backend));
@@ -1685,8 +1933,11 @@ bool ModelManager::fits_compute_backend_capacity(
     const DeviceMemoryRequest& request,
     const std::vector<ggml_tensor*>& required_params) const {
     std::vector<TensorState*> states;
-    return resolve_required_tensor_states(required_params, states, request.compute_backend) &&
-           check_capacity(request, states).fits();
+    if (!resolve_required_tensor_states(required_params, states, request.compute_backend)) {
+        return false;
+    }
+    const auto capacity = check_capacity(request, states);
+    return !capacity.requires_reclamation && capacity.fits();
 }
 
 bool ModelManager::ensure_compute_backend_capacity(
@@ -1708,7 +1959,10 @@ bool ModelManager::ensure_compute_backend_capacity(
         return true;
     }
 
-    auto fits = [&]() { return check_capacity(request, required_states).fits(); };
+    auto fits = [&]() {
+        const auto capacity = check_capacity(request, required_states);
+        return !capacity.requires_reclamation && capacity.fits();
+    };
     if (fits()) {
         return true;
     }
@@ -1748,7 +2002,12 @@ bool ModelManager::ensure_compute_backend_capacity(
             eviction_states.insert(state);
         }
     };
+    size_t last_eviction_count   = 0;
     auto release_eviction_states = [&]() {
+        if (eviction_states.size() == last_eviction_count) {
+            return false;
+        }
+        last_eviction_count = eviction_states.size();
         release_compute_staging_blocks(false, &eviction_states);
         release_params_storage_blocks(false, &eviction_states);
         return fits();
@@ -1786,7 +2045,53 @@ bool ModelManager::ensure_compute_backend_capacity(
         }
     }
 
-    const auto capacity                = check_capacity(request, required_states);
+    // The loader retains mmap buffers after storage blocks are released, so
+    // demoting mapped states would only force duplicate allocations on reload.
+    std::unordered_set<TensorState*> mapped_states;
+    for (const auto& block : params_storage_blocks_) {
+        if (!block->mmap_tensor_stores.empty()) {
+            mapped_states.insert(block->states.begin(), block->states.end());
+        }
+    }
+    std::unordered_set<TensorState*> required_set(required_states.begin(), required_states.end());
+    std::vector<TensorState*> demoted_states;
+    auto revert_demotions = [&]() {
+        for (TensorState* state : demoted_states) {
+            if (state->loaded_to_params_backend) {
+                state->residency_mode = ResidencyMode::ParamBackend;
+            }
+        }
+    };
+    auto can_demote_to_disk = [&](TensorState* state) {
+        return state != nullptr && state->tensor != nullptr && state->tensor->view_src == nullptr &&
+               state->residency_mode == ResidencyMode::ParamBackend &&
+               state->params_backend != nullptr && state->params_backend == state->compute_backend &&
+               state->compute_backend == compute_backend && !sd_backend_is_cpu(state->params_backend) &&
+               state->pin_count == 0 && !state->staged_to_compute_backend && state->loaded_to_params_backend &&
+               protected_states.find(state) == protected_states.end() &&
+               required_set.find(state) == required_set.end() &&
+               mapped_states.find(state) == mapped_states.end() &&
+               !state->sources.empty() && state->applied_lora_epoch == UINT64_MAX &&
+               state->split_buffer_type == nullptr;
+    };
+    for (TensorState* state : global_candidates) {
+        if (!can_demote_to_disk(state)) {
+            continue;
+        }
+        state->residency_mode = ResidencyMode::Disk;
+        demoted_states.push_back(state);
+        add_evictable_state(state);
+        if (release_eviction_states()) {
+            revert_demotions();
+            return true;
+        }
+    }
+    revert_demotions();
+
+    if (check_capacity(request, required_states).fits()) {
+        return true;
+    }
+    const auto capacity                = check_capacity(request, required_states, true);
     const std::string available_device = capacity.available_device_bytes == SIZE_MAX
                                              ? "unknown"
                                              : sd_format("%.2f MB", capacity.available_device_bytes / (1024.0 * 1024.0));
diff --git a/src/model_manager.h b/src/model_manager.h
index 7f8df59c9b..f6d442a8d5 100644
--- a/src/model_manager.h
+++ b/src/model_manager.h
@@ -38,8 +38,7 @@ class ModelManager : public DeviceResidencyManager {
         std::string name;
         ggml_tensor* tensor      = nullptr;
         ModelComponent component = ModelComponent::Count;
-        TensorStorage source;
-        bool has_source                 = false;
+        std::vector<TensorStorage> sources;
         ModelLoader::FileId source_file = 0;
         SDVersion source_version        = VERSION_COUNT;
 
@@ -50,6 +49,7 @@ class ModelManager : public DeviceResidencyManager {
         bool params_follow_compute_backend           = false;
         bool metadata_validated                      = false;
         enum ggml_op usage_op                        = GGML_OP_NONE;
+        bool matmul_only                             = false;  // tagged by its block: may be CPU-repacked
 
         int pin_count = 0;
 
@@ -94,6 +94,7 @@ class ModelManager : public DeviceResidencyManager {
     std::vector<std::unique_ptr<TensorState>> tensor_states_;
     std::map<const ggml_tensor*, TensorState*> tensor_states_by_tensor_;
     mutable std::list<ResolvedTensorStates> resolved_tensor_states_;
+    mutable std::map<std::pair<uintptr_t, ggml_backend_dev_t>, std::pair<size_t, size_t>> device_query_cache_;
     std::vector<std::unique_ptr<ParamsStorageBlock>> params_storage_blocks_;
     std::vector<std::unique_ptr<ComputeStagingBlock>> compute_staging_blocks_;
     std::map<ggml_backend_t, ggml_backend_buffer_type_t> split_buffer_types_;
@@ -113,6 +114,7 @@ class ModelManager : public DeviceResidencyManager {
     int n_threads_                   = 0;
     bool enable_mmap_                = false;
     bool writable_mmap_              = false;
+    bool cpu_repack_allowed_         = true;
     bool segmented_compute_disabled_ = false;
     bool prefetch_disabled_          = false;
 
@@ -138,7 +140,10 @@ class ModelManager : public DeviceResidencyManager {
     bool apply_loras_to_params(const std::vector<TensorState*>& states);
     bool mmap_params(const std::vector<TensorState*>& states,
                      std::vector<ParamsStorageBlock*>& created_storage_blocks);
+    static std::vector<TensorStorage> find_tensor_sources(const TensorState& state, const String2TensorStorage& sources);
+    bool load_tensor_parts(TensorState& state);
     bool can_mmap_storage(const TensorState& state) const;
+    void sync_cpu_repack_tag(TensorState& state) const;
     bool alloc_params_buffers(const std::vector<TensorState*>& states,
                               std::vector<ParamsStorageBlock*>& created_storage_blocks);
     bool load_tensors(const std::vector<TensorState*>& states);
@@ -151,13 +156,15 @@ class ModelManager : public DeviceResidencyManager {
         size_t required_budget_bytes  = 0;
         size_t available_device_bytes = SIZE_MAX;
         size_t available_budget_bytes = SIZE_MAX;
+        bool requires_reclamation     = false;
         bool fits() const {
             return required_device_bytes <= available_device_bytes &&
                    required_budget_bytes <= available_budget_bytes;
         }
     };
     CapacityCheck check_capacity(const DeviceMemoryRequest& request,
-                                 const std::vector<TensorState*>& states) const;
+                                 const std::vector<TensorState*>& states,
+                                 bool log_details = false) const;
 
     ggml_backend_buffer_type_t params_buffer_type_for(const TensorState& state) const;
     ggml_backend_buffer_type_t split_buffer_type_for(const TensorState& state) const;
@@ -198,8 +205,10 @@ class ModelManager : public DeviceResidencyManager {
         segmented_compute_disabled_ = disabled;
     }
     void set_prefetch_disabled(bool disabled) { prefetch_disabled_ = disabled; }
-    void set_enable_mmap(bool enable_mmap) { enable_mmap_ = enable_mmap; }
+    void set_enable_mmap(bool enable_mmap);
     void set_writable_mmap(bool writable_mmap) { writable_mmap_ = writable_mmap; }
+    // Per context: an 'immediately' LoRA merge rewrites weights in place, which a repacked CPU weight cannot take.
+    void set_cpu_repack_allowed(bool allowed);
     void set_common_ignore_tensors(std::set<std::string> ignore_tensors);
     bool prepare_lora_sources(std::vector<LoraSpec>& loras);
     bool set_loras(std::vector<LoraSpec> loras, SDVersion version);
diff --git a/src/model_manager_files.cpp b/src/model_manager_files.cpp
index b5dfe7c7c5..10c7d2e68f 100644
--- a/src/model_manager_files.cpp
+++ b/src/model_manager_files.cpp
@@ -15,6 +15,26 @@ static bool same_tensor_source(const TensorStorage& a, const TensorStorage& b) {
            a.int8_convrot_group_size == b.int8_convrot_group_size;
 }
 
+std::vector<TensorStorage> ModelManager::find_tensor_sources(const TensorState& state, const String2TensorStorage& sources) {
+    auto first = sources.find(state.name);
+    if (first == sources.end()) {
+        return {};
+    }
+    std::vector<TensorStorage> result{first->second};
+    if (state.component == ModelComponent::LoRA ||
+        std::equal(first->second.ne, first->second.ne + GGML_MAX_DIMS, state.tensor->ne)) {
+        return result;
+    }
+    for (size_t i = 1;; ++i) {
+        auto part = sources.find(state.name + "." + std::to_string(i));
+        if (part == sources.end()) {
+            break;
+        }
+        result.push_back(part->second);
+    }
+    return result;
+}
+
 void ModelManager::invalidate_sources(const std::unordered_set<TensorState*>& states) {
     auto affected = states;
     for (const auto& block : params_storage_blocks_) {
@@ -76,20 +96,16 @@ bool ModelManager::set_loader(ModelLoader loader) {
     }
     std::unordered_set<TensorState*> changed;
     for (const auto& state : tensor_states_) {
-        const auto& sources = sources_for(*state);
-        auto source         = sources.find(state->name);
-        const bool found    = source != sources.end();
-        if (found != state->has_source || (found && !same_tensor_source(state->source, source->second)) ||
+        const auto sources = find_tensor_sources(*state, sources_for(*state));
+        if (sources.size() != state->sources.size() ||
+            !std::equal(sources.begin(), sources.end(), state->sources.begin(), same_tensor_source) ||
             (lora_changed && state->component != ModelComponent::LoRA && state->applied_lora_epoch != UINT64_MAX)) {
             changed.insert(state.get());
         }
     }
     invalidate_sources(changed);
     for (auto* state : changed) {
-        const auto& sources = sources_for(*state);
-        auto source         = sources.find(state->name);
-        state->has_source   = source != sources.end();
-        state->source       = state->has_source ? source->second : TensorStorage{};
+        state->sources = find_tensor_sources(*state, sources_for(*state));
     }
     if (lora_changed) {
         ++current_lora_epoch_;
@@ -134,9 +150,8 @@ ModelLoader::FileVersions ModelManager::source_versions(const std::set<ModelComp
             versions[state->source_file] = loader.file_revision(state->source_file);
             continue;
         }
-        auto source = sources.find(state->name);
-        if (source != sources.end()) {
-            versions[source->second.file_id] = source->second.file_revision;
+        for (const auto& source : find_tensor_sources(*state, sources)) {
+            versions[source.file_id] = source.file_revision;
         }
     }
     return versions;
diff --git a/src/model_manager_prefetch.cpp b/src/model_manager_prefetch.cpp
index 1d3041938d..de96d5ae3b 100644
--- a/src/model_manager_prefetch.cpp
+++ b/src/model_manager_prefetch.cpp
@@ -210,7 +210,7 @@ WeightPrefetchResult ModelManager::prefetch_params(
             is_optional_missing_tensor(state->name)) {
             continue;
         }
-        if (state->compute_backend == state->params_backend) {
+        if (state->usage_op == GGML_OP_CUSTOM || state->compute_backend == state->params_backend) {
             needs_synchronous_load = needs_synchronous_load ||
                                      !state->loaded_to_params_backend;
             continue;
@@ -275,6 +275,7 @@ bool ModelManager::activate_prefetched_params(
         [&](TensorState* state) {
             return state == nullptr || should_ignore(*state) ||
                    is_optional_missing_tensor(state->name) ||
+                   state->usage_op == GGML_OP_CUSTOM ||
                    state->compute_backend == state->params_backend ||
                    state->staged_to_compute_backend;
         });
diff --git a/src/name_conversion.cpp b/src/name_conversion.cpp
index 6f1b598e88..6a519ba21f 100644
--- a/src/name_conversion.cpp
+++ b/src/name_conversion.cpp
@@ -105,7 +105,25 @@ std::string convert_open_clip_to_hf_clip_name(std::string name) {
 
 std::string convert_llada2_moe_te_name(std::string name);
 
-std::string convert_cond_stage_model_name(std::string name, std::string prefix) {
+static std::string convert_ming_image_te_name(std::string name) {
+    if (name == "llm.query_tokens") {
+        name = "llm.query_tokens_dict.16x16";
+    }
+    static const std::vector<std::pair<std::string, std::string>> name_map = {
+        {"thinker.", "backbone."},
+        {"attention.", "self_attn."},
+        {"self_attn.dense.", "self_attn.o_proj."},
+        {"gate.proj.", "gate."},
+    };
+    replace_with_name_map(name, name_map);
+    return name;
+}
+
+std::string convert_cond_stage_model_name(std::string name, std::string prefix, SDVersion version) {
+    if (version == VERSION_MING_IMAGE && prefix == "text_encoders." && starts_with(name, "llm.")) {
+        name = convert_ming_image_te_name(name);
+    }
+
     static const std::vector<std::pair<std::string, std::string>> clip_name_map{
         {"transformer.text_projection.weight", "transformer.text_model.text_projection"},
         {"model.text_projection.weight", "transformer.text_model.text_projection"},
@@ -806,6 +824,7 @@ std::string convert_diffusers_dit_to_original_lumina2(std::string name) {
     if (z_image_name_map.empty()) {
         z_image_name_map["all_x_embedder.2-1."]  = "x_embedder.";
         z_image_name_map["all_final_layer.2-1."] = "final_layer.";
+        z_image_name_map["all_x_embedder.16-1."] = "x_embedder.";
 
         // --- transformer blocks ---
         auto add_attention_map = [&](const std::string& prefix, int num) {
@@ -932,6 +951,107 @@ static bool is_diffusers_controlnet_name(const std::string& name) {
     return false;
 }
 
+static std::string convert_diffusers_dit_to_original_pixart(std::string name) {
+    static const std::vector<std::pair<std::string, std::string>> prefix_map = {
+        {"pos_embed.proj.", "x_embedder.proj."},
+        {"adaln_single.emb.timestep_embedder.linear_1.", "t_embedder.mlp.0."},
+        {"adaln_single.emb.timestep_embedder.linear_2.", "t_embedder.mlp.2."},
+        {"adaln_single.emb.resolution_embedder.linear_1.", "csize_embedder.mlp.0."},
+        {"adaln_single.emb.resolution_embedder.linear_2.", "csize_embedder.mlp.2."},
+        {"adaln_single.emb.aspect_ratio_embedder.linear_1.", "ar_embedder.mlp.0."},
+        {"adaln_single.emb.aspect_ratio_embedder.linear_2.", "ar_embedder.mlp.2."},
+        {"adaln_single.linear.", "t_block.1."},
+        {"caption_projection.linear_1.", "y_embedder.y_proj.fc1."},
+        {"caption_projection.linear_2.", "y_embedder.y_proj.fc2."},
+        {"proj_out.", "final_layer.linear."},
+    };
+    for (const auto& entry : prefix_map) {
+        if (starts_with(name, entry.first)) {
+            return entry.second + name.substr(entry.first.size());
+        }
+    }
+    if (name == "scale_shift_table") {
+        return "final_layer.scale_shift_table";
+    }
+    const std::string block_prefix = "transformer_blocks.";
+    if (!starts_with(name, block_prefix)) {
+        return name;
+    }
+    const size_t block_end = name.find('.', block_prefix.size());
+    if (block_end == std::string::npos) {
+        return name;
+    }
+    const std::string prefix                                                = "blocks." + name.substr(block_prefix.size(), block_end - block_prefix.size()) + ".";
+    name                                                                    = name.substr(block_end + 1);
+    static const std::vector<std::pair<std::string, std::string>> block_map = {
+        {"attn1.to_q.", "attn.qkv."},
+        {"attn1.to_out.0.", "attn.proj."},
+        {"attn2.to_q.", "cross_attn.q_linear."},
+        {"attn2.to_k.", "cross_attn.kv_linear."},
+        {"attn2.to_out.0.", "cross_attn.proj."},
+        {"ff.net.0.proj.", "mlp.fc1."},
+        {"ff.net.2.", "mlp.fc2."},
+    };
+    for (const auto& entry : block_map) {
+        if (starts_with(name, entry.first)) {
+            return prefix + entry.second + name.substr(entry.first.size());
+        }
+    }
+    static const std::vector<std::pair<std::string, std::string>> part_map = {
+        {"attn1.to_k.weight", "attn.qkv.weight.1"},
+        {"attn1.to_k.bias", "attn.qkv.bias.1"},
+        {"attn1.to_v.weight", "attn.qkv.weight.2"},
+        {"attn1.to_v.bias", "attn.qkv.bias.2"},
+        {"attn2.to_v.weight", "cross_attn.kv_linear.weight.1"},
+        {"attn2.to_v.bias", "cross_attn.kv_linear.bias.1"},
+    };
+    for (const auto& entry : part_map) {
+        if (name == entry.first || starts_with(name, entry.first + ".")) {
+            return prefix + entry.second + name.substr(entry.first.size());
+        }
+    }
+    return prefix + name;
+}
+
+static std::string convert_moss_image_dit_name(std::string name) {
+    static const std::vector<std::pair<std::string, std::string>> name_map = {
+        {".proj.q.weight", ".proj.qkv.weight"},
+        {".proj.k.weight", ".proj.qkv.weight.1"},
+        {".proj.v.weight", ".proj.qkv.weight.2"},
+    };
+    replace_with_name_map(name, name_map);
+    return name;
+}
+
+static std::string convert_ming_image_dit_name(std::string name) {
+    static const std::vector<std::pair<std::string, std::string>> name_map = {
+        {"all_x_embedder.2-1.", "x_embedder."},
+        {"all_final_layer.2-1.", "final_layer."},
+        {"attention.norm_q.", "attention.q_norm."},
+        {"attention.norm_k.", "attention.k_norm."},
+    };
+    replace_with_name_map(name, name_map);
+    return name;
+}
+
+std::string convert_diffusers_dit_to_original_minimax_h3(std::string name) {
+    static const std::vector<std::pair<std::string, std::string>> prefix_map = {
+        {"transformer_blocks.", "blocks."},
+        {"token_refiner.refiner_blocks.", "token_refiner.blocks."},
+    };
+    static const std::vector<std::pair<std::string, std::string>> name_map = {
+        {".attn.to_q.weight", ".attn.qkv_proj.weight"},
+        {".attn.to_k.weight", ".attn.qkv_proj.weight.1"},
+        {".attn.to_v.weight", ".attn.qkv_proj.weight.2"},
+        {".attn.to_out.0.", ".attn.out_proj."},
+        {".ff.net.0.proj.", ".mlp.fc1."},
+        {".ff.net.2.", ".mlp.fc2."},
+    };
+    replace_with_prefix_map(name, prefix_map);
+    replace_with_name_map(name, name_map);
+    return name;
+}
+
 std::string convert_diffusion_model_name(std::string name, std::string prefix, SDVersion version) {
     if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) {
         name = convert_diffusers_unet_to_original_sd1(name);
@@ -943,7 +1063,11 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
         name = convert_diffusers_dit_to_original_flux(name);
     } else if (sd_version_is_hunyuan_video(version)) {
         name = convert_hunyuan_video_to_original_flux(name);
-    } else if (sd_version_is_z_image(version)) {
+    } else if (version == VERSION_MOSS_IMAGE) {
+        name = convert_moss_image_dit_name(name);
+    } else if (version == VERSION_MING_IMAGE) {
+        name = convert_ming_image_dit_name(name);
+    } else if (sd_version_is_z_image(version) || sd_version_is_z_image_l2p(version)) {
         name = convert_diffusers_dit_to_original_lumina2(name);
     } else if (sd_version_is_llada_image(version)) {
         name = convert_diffusers_dit_to_original_llada_image(name);
@@ -951,6 +1075,10 @@ std::string convert_diffusion_model_name(std::string name, std::string prefix, S
         name = convert_other_dit_to_original_anima(name);
     } else if (sd_version_is_krea2(version)) {
         name = convert_diffusers_dit_to_original_krea2(name);
+    } else if (sd_version_is_pixart(version)) {
+        name = convert_diffusers_dit_to_original_pixart(name);
+    } else if (sd_version_is_minimax_h3(version)) {
+        name = convert_diffusers_dit_to_original_minimax_h3(name);
     }
     return name;
 }
@@ -1349,9 +1477,8 @@ bool is_first_stage_model_name(const std::string& name) {
 }
 
 static std::string convert_esrgan_tensor_name(std::string name) {
-    static std::unordered_map<std::string, std::string> esrgan_name_map;
-
-    if (esrgan_name_map.empty()) {
+    static const auto esrgan_name_map = [] {
+        std::unordered_map<std::string, std::string> esrgan_name_map;
         esrgan_name_map["model.0."] = "conv_first.";
 
         constexpr int max_num_blocks = 64;
@@ -1377,7 +1504,8 @@ static std::string convert_esrgan_tensor_name(std::string name) {
         esrgan_name_map["model.7."]  = "conv_last.";
         esrgan_name_map["model.8."]  = "conv_hr.";
         esrgan_name_map["model.10."] = "conv_last.";
-    }
+        return esrgan_name_map;
+    }();
 
     replace_with_prefix_map(name, esrgan_name_map);
     return name;
@@ -1567,6 +1695,26 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
 
     replace_with_prefix_map(name, prefix_map);
 
+    if (version == VERSION_MOSS_IMAGE) {
+        const std::string te_prefix = "text_encoders.llm.";
+        if (starts_with(name, te_prefix + "blk.")) {
+            static const std::vector<std::pair<std::string, std::string>> qwen35_name_map = {
+                {"attn_post_norm.", "post_attention_layernorm."},
+                {"post_attention_norm.", "post_attention_layernorm."},
+                {"attn_qkv.", "linear_attn.in_proj_qkv."},
+                {"attn_gate.", "linear_attn.in_proj_z."},
+                {"ssm_conv1d.", "linear_attn.conv1d."},
+                {"ssm_dt.bias", "linear_attn.dt_bias"},
+                {"ssm_norm.", "linear_attn.norm."},
+                {"ssm_out.", "linear_attn.out_proj."},
+                {"ssm_alpha.", "linear_attn.in_proj_a."},
+                {"ssm_beta.", "linear_attn.in_proj_b."},
+                {"ssm_a", "linear_attn.A"},
+            };
+            replace_with_name_map(name, qwen35_name_map);
+        }
+    }
+
     if (version == VERSION_QWEN_IMAGE_2_1 || sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_mage_flow(version) || sd_version_is_minimax_h3(version)) {
         const std::string hf_vision_prefix = "text_encoders.llm.model.visual.";
         if (starts_with(name, hf_vision_prefix)) {
@@ -1604,7 +1752,7 @@ std::string convert_tensor_name(std::string name, SDVersion version) {
     {
         for (const auto& prefix : cond_stage_model_prefix_vec) {
             if (starts_with(name, prefix)) {
-                name = convert_cond_stage_model_name(name.substr(prefix.size()), prefix);
+                name = convert_cond_stage_model_name(name.substr(prefix.size()), prefix, version);
                 name = prefix + name;
                 break;
             }
diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp
index e41c1151a8..82b40473a5 100644
--- a/src/pipeline/diffusion_engine.cpp
+++ b/src/pipeline/diffusion_engine.cpp
@@ -4,9 +4,11 @@
 #include <cinttypes>
 #include <cmath>
 #include <cstdlib>
+#include <cstring>
 #include <list>
 #include <mutex>
 #include <set>
+#include <tuple>
 #include <type_traits>
 #include <unordered_set>
 #include <utility>
@@ -17,6 +19,8 @@
 #include "core/ggml_runner.h"
 #include "core/ggml_tensor_utils.h"
 #include "core/layer_split_partition.h"
+#include "core/probe/conv_direct.h"
+#include "core/probe/flash_attn.h"
 #include "model.h"
 
 #include "core/rng.hpp"
@@ -29,6 +33,7 @@
 #include "stable-diffusion.h"
 
 #include "conditioning/conditioner.hpp"
+#include "conditioning/conditioning_cache.h"
 #include "core/backend_fit.h"
 #include "extensions/generation_extension.h"
 #include "model/adapter/ip_adapter.hpp"
@@ -103,6 +108,12 @@ const char* model_version_to_str[] = {
     "SenseNova U1.5",
     "LLaDA-Image",
     "ESRGAN",
+    "PixArt",
+    "Ming-Image",
+    "Z-Image L2P",
+    "Iris-3B",
+    "Flux.3 Image Klein",
+    "MOSS-Image",
 };
 
 static_assert(VERSION_COUNT == sizeof(model_version_to_str) / sizeof(model_version_to_str[0]),
@@ -123,6 +134,18 @@ void calculate_alphas_cumprod(float* alphas_cumprod,
     }
 }
 
+void calculate_alphas_cumprod_linear_beta(float* alphas_cumprod,
+                                          float beta_start,
+                                          float beta_end,
+                                          int timesteps = TIMESTEPS) {
+    float product = 1.0f;
+    for (int i = 0; i < timesteps; i++) {
+        float beta = beta_start + (beta_end - beta_start) * ((float)i / (timesteps - 1));
+        product *= 1.0f - beta;
+        alphas_cumprod[i] = product;
+    }
+}
+
 template <typename T, typename = void>
 struct has_set_runtime_backends : std::false_type {};
 template <typename T>
@@ -135,6 +158,7 @@ static_assert(std::atomic<sd_cancel_mode_t>::is_always_lock_free,
 
 StableDiffusionGGML::StableDiffusionGGML()
     : rng(std::make_shared<PhiloxRNG>()),
+      conditioning_cache_(std::make_unique<ConditioningCache>()),
       denoiser(std::make_shared<CompVisDenoiser>()) {}
 
 StableDiffusionGGML::~StableDiffusionGGML() = default;
@@ -204,6 +228,8 @@ void StableDiffusionGGML::end_runners() {
 }
 
 bool StableDiffusionGGML::reset_runners(const RunnerGroups& groups) {
+    conditioning_cache_->clear();
+    conditioning_loras_.clear();
     end_runners();
     clear_lora_adapters();
     runtime_lora_models.clear();
@@ -661,6 +687,10 @@ void StableDiffusionGGML::refresh_compvis_denoiser_sigmas() {
     std::vector<float> alphas_cumprod(TIMESTEPS);
     if (file_alphas_cumprod.size() == TIMESTEPS) {
         alphas_cumprod = file_alphas_cumprod;
+    } else if (sd_version_is_pixart(version)) {
+        // PixArt checkpoints train with a linear beta schedule (0.0001 -> 0.02)
+        // instead of the scaled_linear schedule used by SD1.x/SDXL.
+        calculate_alphas_cumprod_linear_beta(alphas_cumprod.data(), 0.0001f, 0.02f);
     } else {
         calculate_alphas_cumprod(alphas_cumprod.data());
     }
@@ -915,6 +945,26 @@ bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) {
             return false;
         }
     }
+    if (sd_ctx_params->conditioning_cache_size < 0) {
+        LOG_ERROR("conditioning_cache_size must be non-negative");
+        return false;
+    }
+    const std::string flash_mode = SAFE_STR(sd_ctx_params->flash_attn_mode);
+    std::string flash_error;
+    if (!sd_parse_flash_attention_assignment(flash_mode, &flash_attn_assignment, &flash_error)) {
+        LOG_ERROR("invalid flash_attn_mode '%s': %s", flash_mode.c_str(), flash_error.c_str());
+        return false;
+    }
+    const std::string conv_mode = SAFE_STR(sd_ctx_params->conv_direct_mode);
+    std::string conv_error;
+    if (!sd_parse_conv_direct_assignment(conv_mode, &conv_direct_assignment, &conv_error)) {
+        LOG_ERROR("invalid conv_direct_mode '%s': %s", conv_mode.c_str(), conv_error.c_str());
+        return false;
+    }
+    flash_attn_by_backend.clear();
+    conv2d_direct_by_backend.clear();
+    conv3d_direct_by_backend.clear();
+    conditioning_cache_->set_capacity(static_cast<size_t>(sd_ctx_params->conditioning_cache_size));
     auto configuration = std::make_unique<ModelConfig>(*sd_ctx_params);
     n_threads          = sd_ctx_params->n_threads;
     tensor_executor    = std::make_unique<sd::ParallelExecutor>(n_threads > 0 ? n_threads : sd_get_num_physical_cores());
@@ -1009,9 +1059,15 @@ void StableDiffusionGGML::configure_weight_loading() {
     const auto& model_loader  = model_manager->loader();
     const auto wtype_stat     = model_loader.get_wtype_stat();
     bool have_int8_tensorwise = false;
+    bool have_quant_type_rule = false;
     for (const auto& [_, tensor_storage] : model_loader.get_tensor_storage_map()) {
         if (tensor_storage.is_int8_tensorwise) {
             have_int8_tensorwise = true;
+        }
+        if (tensor_storage.expected_type < GGML_TYPE_COUNT && ggml_is_quantized(tensor_storage.expected_type)) {
+            have_quant_type_rule = true;
+        }
+        if (have_int8_tensorwise && have_quant_type_rule) {
             break;
         }
     }
@@ -1028,7 +1084,8 @@ void StableDiffusionGGML::configure_weight_loading() {
         const bool params_offloaded      = params_backend_for(SDBackendModule::DIFFUSION) != backend_for(SDBackendModule::DIFFUSION);
         const bool streaming_constrained = params_offloaded ||
                                            backend_manager.params_backend_is_disk(SDBackendModule::DIFFUSION);
-        if (have_quantized_weight || streaming_constrained || row_split_active()) {
+
+        if (have_quant_type_rule || have_quantized_weight || streaming_constrained || row_split_active()) {
             apply_lora_immediately = false;
         } else {
             apply_lora_immediately = true;
@@ -1051,6 +1108,9 @@ void StableDiffusionGGML::configure_weight_loading() {
         apply_lora_immediately = false;
     }
 
+    // Merging a LoRA rewrites weights in place, which a repacked CPU weight cannot take.
+    model_manager->set_cpu_repack_allowed(!apply_lora_immediately);
+
     bool needs_writable_mmap = enable_mmap && apply_lora_immediately;
     model_manager->set_writable_mmap(needs_writable_mmap);
     if (enable_mmap && apply_lora_immediately) {
@@ -1152,37 +1212,147 @@ bool StableDiffusionGGML::build_extension_runners() {
     return true;
 }
 
+bool StableDiffusionGGML::flash_attention_for(SDBackendModule module) {
+    const auto& params     = config_->params;
+    const std::string mode = flash_attn_assignment.get(module);
+    if (mode == "on") {
+        return true;
+    }
+    if (mode == "off") {
+        return false;
+    }
+    if (strlen(SAFE_STR(params.flash_attn_mode)) == 0) {
+        if (params.flash_attn) {
+            return true;
+        }
+        if (params.diffusion_flash_attn) {
+            return module == SDBackendModule::DIFFUSION;
+        }
+    }
+    auto backends    = backend_manager.runtime_backends(module);
+    bool enabled     = !backends.empty();
+    bool any_enabled = false;
+    for (auto backend : backends) {
+        auto it = flash_attn_by_backend.find(backend);
+        if (it == flash_attn_by_backend.end()) {
+            it = flash_attn_by_backend.emplace(backend, sd_probe_flash_attention(backend, n_threads)).first;
+        }
+        enabled     = enabled && it->second;
+        any_enabled = any_enabled || it->second;
+    }
+    if (any_enabled && !enabled) {
+        LOG_INFO("flash attention auto: %s -> off (compute devices disagree)", sd_backend_module_name(module));
+    }
+    return enabled;
+}
+
+bool StableDiffusionGGML::conv_direct_for(SDBackendModule module, bool conv3d) {
+    const auto& params     = config_->params;
+    const std::string mode = conv_direct_assignment.get(module);
+    if (mode == "on" || mode == "off") {
+        return mode == "on";
+    }
+    if (strlen(SAFE_STR(params.conv_direct_mode)) == 0 &&
+        (module == SDBackendModule::VAE ? params.vae_conv_direct : params.diffusion_conv_direct)) {
+        return true;
+    }
+    auto& by_backend = conv3d ? conv3d_direct_by_backend : conv2d_direct_by_backend;
+    auto backends    = backend_manager.runtime_backends(module);
+    bool enabled     = !backends.empty();
+    bool any_enabled = false;
+    for (auto backend : backends) {
+        auto it = by_backend.find(backend);
+        if (it == by_backend.end()) {
+            it = by_backend.emplace(backend, sd_probe_conv_direct(backend, conv3d)).first;
+        }
+        enabled     = enabled && it->second;
+        any_enabled = any_enabled || it->second;
+    }
+    if (any_enabled && !enabled) {
+        LOG_INFO("conv%dd direct auto: %s -> off (compute devices disagree)", conv3d ? 3 : 2, sd_backend_module_name(module));
+    }
+    return enabled;
+}
+
 bool StableDiffusionGGML::validate_and_load_runners() {
     const auto* sd_ctx_params   = &config_->params;
     const bool use_tae          = uses_tae();
     const bool tae_preview_only = this->tae_preview_only();
-    if (sd_ctx_params->flash_attn) {
-        LOG_INFO("Using flash attention");
-        cond_stage_model->set_flash_attention_enabled(true);
-        if (clip_vision) {
-            clip_vision->set_flash_attention_enabled(true);
-        }
+    cond_stage_model->set_flash_attention_enabled(flash_attention_for(SDBackendModule::TE));
+    if (clip_vision) {
+        clip_vision->set_flash_attention_enabled(flash_attention_for(SDBackendModule::CLIP_VISION));
+    }
+    bool vae_fa = false;
+    if (first_stage_model || preview_vae || audio_vae_model) {
+        const bool enabled = flash_attention_for(SDBackendModule::VAE);
+        vae_fa             = enabled;
         if (first_stage_model) {
-            first_stage_model->set_flash_attention_enabled(true);
+            first_stage_model->set_flash_attention_enabled(enabled);
         }
         if (preview_vae) {
-            preview_vae->set_flash_attention_enabled(true);
+            preview_vae->set_flash_attention_enabled(enabled);
+        }
+        if (audio_vae_model) {
+            audio_vae_model->set_flash_attention_enabled(enabled);
         }
     }
-
-    if (sd_ctx_params->flash_attn || sd_ctx_params->diffusion_flash_attn) {
-        LOG_INFO("Using flash attention in the diffusion model");
-        diffusion_model->set_flash_attention_enabled(true);
-        if (high_noise_diffusion_model) {
-            high_noise_diffusion_model->set_flash_attention_enabled(true);
+    const bool diffusion_fa = flash_attention_for(SDBackendModule::DIFFUSION);
+    diffusion_model->set_flash_attention_enabled(diffusion_fa);
+    if (high_noise_diffusion_model) {
+        high_noise_diffusion_model->set_flash_attention_enabled(diffusion_fa);
+    }
+    if (ip_adapter) {
+        ip_adapter->set_flash_attention_enabled(diffusion_fa);
+    }
+    if (control_net) {
+        control_net->set_flash_attention_enabled(flash_attention_for(SDBackendModule::CONTROL_NET));
+    }
+    if (audio_encoder) {
+        audio_encoder->set_flash_attention_enabled(flash_attention_for(SDBackendModule::AUDIO_ENCODER));
+    }
+    // The H3 video VAE decoder is a ViT whose f32 L x L attention scores dominate decode time, so on CUDA and ROCm
+    // --diffusion-fa covers it too, without switching the text encoder. It decoded slower on Vulkan, so other
+    // backends keep the old path unless SD_H3_VAE_FLASH_ATTN=1; SD_H3_VAE_FLASH_ATTN=0 restores it everywhere.
+    // Keyed on the resolved per-module state: the CLI now spells --diffusion-fa as --flash-attn off,diffusion=on and
+    // never sets the legacy diffusion_flash_attn flag. An explicit vae= assignment is the user's choice and wins.
+    const bool vae_fa_assigned = std::string(SAFE_STR(sd_ctx_params->flash_attn_mode)).find("vae=") != std::string::npos;
+    if (diffusion_fa && !vae_fa && !vae_fa_assigned && first_stage_model && sd_version_is_minimax_h3(version)) {
+        const char* env                 = getenv("SD_H3_VAE_FLASH_ATTN");
+        ggml_backend_t vae_backend      = backend_for(SDBackendModule::VAE);
+        const bool fa_faster_by_default = sd_backend_is(vae_backend, "CUDA") || sd_backend_is(vae_backend, "ROCm");
+        if (env != nullptr ? strcmp(env, "0") != 0 : fa_faster_by_default) {
+            LOG_INFO("Using flash attention in the MiniMax-H3 video VAE");
+            first_stage_model->set_flash_attention_enabled(true);
         }
     }
     if (sd_ctx_params->sage_attn && !set_sage_attention_enabled(true)) {
         return false;
     }
+    const auto set_conv_direct = [this](SDBackendModule module, const std::vector<GGMLRunner*>& runners) {
+        const bool conv2d = conv_direct_for(module, false);
+        const bool conv3d = conv_direct_for(module, true);
+        for (auto runner : runners) {
+            if (runner != nullptr) {
+                runner->set_conv2d_direct_enabled(conv2d);
+                runner->set_conv3d_direct_enabled(conv3d);
+            }
+        }
+    };
+    set_conv_direct(SDBackendModule::DIFFUSION, {diffusion_model.get(), high_noise_diffusion_model.get()});
+    if (first_stage_model || preview_vae || audio_vae_model) {
+        set_conv_direct(SDBackendModule::VAE, {first_stage_model.get(), preview_vae.get(), audio_vae_model.get()});
+    }
+    if (control_net) {
+        set_conv_direct(SDBackendModule::CONTROL_NET, {control_net.get()});
+    }
     LOG_VERBOSE("validating model metadata");
 
     std::set<std::string> ignore_tensors;
+    if (version == VERSION_MING_IMAGE) {
+        ignore_tensors.insert("text_encoders.llm.vision.");
+        ignore_tensors.insert("text_encoders.llm.linear_proj.");
+        ignore_tensors.insert("text_encoders.llm.backbone.lm_head.");
+    }
     if (use_tae && !tae_preview_only) {
         ignore_tensors.insert("first_stage_model.");
     }
@@ -1343,18 +1513,22 @@ bool StableDiffusionGGML::build_denoiser() {
                    sd_version_is_anima(version) ||
                    sd_version_is_ernie_image(version) ||
                    sd_version_is_z_image(version) ||
+                   sd_version_is_z_image_l2p(version) ||
+                   version == VERSION_MING_IMAGE ||
                    sd_version_is_llada_image(version) ||
                    sd_version_is_boogu_image(version) ||
                    sd_version_is_pid(version) ||
-                   sd_version_is_ideogram4(version)) {
+                   sd_version_is_ideogram4(version) ||
+                   version == VERSION_IRIS ||
+                   version == VERSION_MOSS_IMAGE) {
             pred_type = FLOW_PRED;
-            if (sd_version_is_wan(version)) {
+            if (sd_version_is_wan(version) || version == VERSION_MOSS_IMAGE) {
                 default_flow_shift = 5.f;
             } else if (sd_version_is_hunyuan_video(version)) {
                 default_flow_shift = 7.f;
             } else if (sd_version_is_minimax_h3(version)) {
                 default_flow_shift = 12.f;
-            } else if (sd_version_is_ernie_image(version)) {
+            } else if (sd_version_is_ernie_image(version) || version == VERSION_IRIS) {
                 default_flow_shift = 4.f;
             } else if (sd_version_is_pid(version)) {
                 default_flow_shift = 1.5f;
@@ -1364,11 +1538,16 @@ bool StableDiffusionGGML::build_denoiser() {
                 default_flow_shift = 3.16f;
             } else if (sd_version_is_mage_flow(version)) {
                 default_flow_shift = 6.f;
+            } else if (version == VERSION_MING_IMAGE) {
+                default_flow_shift = INFINITY;
             } else if (sd_version_is_llada_image(version)) {
                 default_flow_shift = 1.0f;  // unused: LLADA_IMAGE_SCHEDULER builds a fixed grid
             } else {
                 default_flow_shift = 3.f;
             }
+        } else if (sd_version_is_flux3(version)) {
+            pred_type          = FLUX_FLOW_PRED;
+            default_flow_shift = 6.93f;
         } else if (sd_version_is_flux(version) ||
                    sd_version_is_flux2(version) ||
                    sd_version_is_longcat(version) ||
@@ -1424,6 +1603,8 @@ bool StableDiffusionGGML::build_denoiser() {
             } else if (sd_version_is_minimax_h3(version)) {
                 LOG_INFO("running in MiniMax H3 AV FLOW mode");
                 denoiser = std::make_shared<H3AVFlowDenoiser>(default_flow_shift, 3.f, get_latent_channel());
+            } else if (version == VERSION_MING_IMAGE) {
+                denoiser = std::make_shared<MingImageFlowDenoiser>();
             } else {
                 LOG_INFO("running in FLOW mode");
                 denoiser = std::make_shared<DiscreteFlowDenoiser>();
@@ -1432,7 +1613,11 @@ bool StableDiffusionGGML::build_denoiser() {
         }
         case FLUX_FLOW_PRED: {
             LOG_INFO("running in Flux FLOW mode");
-            denoiser = std::make_shared<FluxFlowDenoiser>();
+            if (sd_version_is_flux3(version)) {
+                denoiser = std::make_shared<Flux3FlowDenoiser>();
+            } else {
+                denoiser = std::make_shared<FluxFlowDenoiser>();
+            }
             break;
         }
         case SEFI_FLOW_PRED: {
@@ -1486,7 +1671,7 @@ bool StableDiffusionGGML::build_runners(const RunnerGroups& groups) {
     LOG_INFO("Diffusion model weight type stat: %s", wtype_stat_to_str(diffusion_model_wtype_stat).c_str());
     LOG_INFO("VAE weight type stat:             %s", wtype_stat_to_str(vae_wtype_stat).c_str());
 
-    LOG_VERBOSE("ggml tensor size = %d bytes", (int)sizeof(ggml_tensor));
+    LOG_DEBUG("ggml tensor size = %d bytes", (int)sizeof(ggml_tensor));
 
     configure_weight_loading();
     for (auto group : groups) {
@@ -1765,8 +1950,19 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun
     int64_t t0 = ggml_time_ms();
     end_runners();
     clear_lora_adapters();
-    if (!model_manager->prepare_lora_sources(all_loras))
+    if (!model_manager->prepare_lora_sources(all_loras)) {
+        conditioning_cache_->clear();
         return false;
+    }
+    if (!std::equal(all_loras.begin(), all_loras.end(),
+                    conditioning_loras_.begin(), conditioning_loras_.end(),
+                    [](const ModelManager::LoraSpec& a, const ModelManager::LoraSpec& b) {
+                        return a.file_id == b.file_id && a.file_revision == b.file_revision &&
+                               a.multiplier == b.multiplier && a.is_high_noise == b.is_high_noise &&
+                               a.tensor_name_prefix_filter == b.tensor_name_prefix_filter;
+                    })) {
+        conditioning_cache_->clear();
+    }
     runtime_lora_models.erase(std::remove_if(runtime_lora_models.begin(), runtime_lora_models.end(), [&](const RuntimeLora& entry) {
                                   return std::none_of(all_loras.begin(), all_loras.end(), [&](const ModelManager::LoraSpec& spec) {
                                       return entry.matches(spec);
@@ -1776,6 +1972,7 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun
     const bool success = apply_lora_immediately ? apply_loras_immediately(all_loras)
                                                 : apply_loras_at_runtime(all_loras);
     if (!success) {
+        conditioning_cache_->clear();
         clear_lora_adapters();
         runtime_lora_models.clear();
         return false;
@@ -1785,9 +1982,14 @@ bool StableDiffusionGGML::apply_loras(const sd_lora_t* loras, uint32_t lora_coun
     if (!all_loras.empty()) {
         LOG_INFO("apply_loras completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
     }
+    conditioning_loras_ = std::move(all_loras);
     return true;
 }
 
+SDCondition StableDiffusionGGML::get_learned_condition(const ConditionerParams& params) {
+    return conditioning_cache_->get(*cond_stage_model, n_threads, params);
+}
+
 void StableDiffusionGGML::reset_generation_extensions() {
     for (auto& extension : generation_extensions) {
         extension->reset_runtime_condition();
@@ -1972,6 +2174,8 @@ void StableDiffusionGGML::preview_image(int step,
         int patch_sz                     = 1;
         const float(*latent_rgb_proj)[3] = nullptr;
         float* latent_rgb_bias           = nullptr;
+        const float* latent_alpha_proj   = nullptr;
+        float latent_alpha_bias          = 1.f;
 
         if (channels == 128) {
             if (sd_version_uses_flux2_vae(version)) {
@@ -1985,6 +2189,16 @@ void StableDiffusionGGML::preview_image(int step,
                 LOG_WARN("No latent to RGB projection known for this model");
                 return;
             }
+        } else if (channels == 64) {
+            if (version == VERSION_QWEN_IMAGE_2_1) {
+                latent_rgb_proj   = qwen21_latent_rgb_proj;
+                latent_rgb_bias   = qwen21_latent_rgb_bias;
+                latent_alpha_proj = qwen21_latent_alpha_proj;
+                latent_alpha_bias = qwen21_latent_alpha_bias;
+            } else {
+                LOG_WARN("No latent to RGB projection known for this model");
+                return;
+            }
         } else if (channels == 48) {
             if (sd_version_is_wan(version)) {
                 latent_rgb_proj = wan_22_latent_rgb_proj;
@@ -2035,13 +2249,14 @@ void StableDiffusionGGML::preview_image(int step,
         uint32_t img_width  = static_cast<uint32_t>(_latents.shape()[0]) * patch_sz;
         uint32_t img_height = static_cast<uint32_t>(_latents.shape()[1]) * patch_sz;
 
-        uint8_t* data = (uint8_t*)malloc(frames * img_width * img_height * 3 * sizeof(uint8_t));
+        uint32_t img_channels = latent_alpha_proj != nullptr ? 4 : 3;
+        uint8_t* data         = (uint8_t*)malloc(frames * img_width * img_height * img_channels * sizeof(uint8_t));
         GGML_ASSERT(data != nullptr);
-        preview_latent_video(data, _latents, latent_rgb_proj, latent_rgb_bias, patch_sz);
+        preview_latent_video(data, _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias);
         sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t));
         GGML_ASSERT(images != nullptr);
         for (uint32_t i = 0; i < frames; i++) {
-            images[i] = {img_width, img_height, 3, data + i * img_width * img_height * 3};
+            images[i] = {img_width, img_height, img_channels, data + i * img_width * img_height * img_channels};
         }
         step_callback(step, frames, images, is_noisy, step_callback_data);
         free(data);
@@ -2104,7 +2319,7 @@ std::vector<float> StableDiffusionGGML::prepare_sample_timesteps(float sigma,
     if (version == VERSION_HIDREAM_O1) {
         return std::vector<float>{1.0f - (t / static_cast<float>(TIMESTEPS))};
     }
-    if (sd_version_is_z_image(version) || sd_version_is_ideogram4(version)) {
+    if (sd_version_is_z_image(version) || sd_version_is_z_image_l2p(version) || sd_version_is_ideogram4(version) || version == VERSION_MING_IMAGE) {
         return std::vector<float>{1000.f - t};
     }
     return std::vector<float>{t};
@@ -2155,6 +2370,27 @@ void StableDiffusionGGML::report_sample_progress(int step,
     }
 }
 
+static sd::Tensor<float> batch_two_condition_tensors(const sd::Tensor<float>& a, const sd::Tensor<float>& b) {
+    if (a.empty() || b.empty() || a.dim() != b.dim()) {
+        return {};
+    }
+    if (a.dim() == 1) {
+        if (a.shape() != b.shape()) {
+            return {};
+        }
+        auto batched = sd::ops::concat(a, b, 0);
+        batched.reshape_({a.shape()[0], 2});
+        return batched;
+    }
+    const int64_t batch_dim = a.dim() - 1;
+    for (int64_t d = 0; d < batch_dim; d++) {
+        if (a.shape()[d] != b.shape()[d]) {
+            return {};
+        }
+    }
+    return sd::ops::concat(a, b, static_cast<size_t>(batch_dim));
+}
+
 void StableDiffusionGGML::compute_sample_controls(const sd::Tensor<float>& control_image,
                                                   const sd::Tensor<float>& noised_input,
                                                   const sd::Tensor<float>& timesteps_tensor,
@@ -2216,6 +2452,15 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
     };
     RunnerEndOnExit sample_diffusion_runner_end{work_diffusion_model.get()};
 
+    // These inputs are immutable for this sampling run. Extensions may replace or
+    // modify them per step, so those paths need an explicit stability contract first.
+    const bool cache_qwen_prefix = version == VERSION_QWEN_IMAGE_2_1 &&
+                                   std::none_of(generation_extensions.begin(), generation_extensions.end(),
+                                                [](const auto& extension) { return extension->is_enabled(); });
+    using QwenPrefixInputs = std::tuple<const sd::Tensor<float>*, const sd::Tensor<int32_t>*,
+                                        const std::vector<sd::Tensor<float>>*>;
+    std::vector<QwenPrefixInputs> qwen_prefix_inputs;
+
     RunnerEndOnExit sample_control_runner_end{!control_image.empty() && control_net != nullptr ? control_net.get() : nullptr};
 
     const bool apply_denoise_mask = !denoise_mask.empty() &&
@@ -2227,6 +2472,40 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
     float img_cfg_scale = guidance.img_cfg;
     float slg_scale     = guidance.slg.scale;
     bool slg_uncond     = sd::guidance::parse_skip_layer_guidance_uncond_arg(extra_sample_args);
+    const sd::guidance::MossUncondSkipParams moss_uncond_skip =
+        version == VERSION_MOSS_IMAGE ? sd::guidance::parse_moss_uncond_skip_args(extra_sample_args)
+                                      : sd::guidance::MossUncondSkipParams{};
+    // Sigma bounds of the MOSS window (ComfyUI ModelSamplingDiscreteFlow.percent_to_sigma, multiplier 1).
+    float moss_skip_sigma_high = INFINITY;
+    float moss_skip_sigma_low  = -INFINITY;
+    if (version == VERSION_MOSS_IMAGE && moss_uncond_skip.enabled && moss_uncond_skip.windowed()) {
+        auto flow_denoiser = std::dynamic_pointer_cast<DiscreteFlowDenoiser>(denoiser);
+        float shift        = flow_denoiser ? flow_denoiser->shift : 5.0f;
+        auto percent_to_sigma = [shift](float percent) {
+            if (percent <= 0.0f) {
+                return 1.0f;
+            }
+            if (percent >= 1.0f) {
+                return 0.0f;
+            }
+            return time_snr_shift(shift, 1.0f - percent);
+        };
+        moss_skip_sigma_high = percent_to_sigma(moss_uncond_skip.start);
+        moss_skip_sigma_low  = percent_to_sigma(moss_uncond_skip.end);
+        LOG_INFO("MOSS uncond skip window: sigma-percent %.4f..%.4f (sigma %.4f..%.4f, shift %.3f)",
+                 moss_uncond_skip.start, moss_uncond_skip.end, moss_skip_sigma_low, moss_skip_sigma_high, shift);
+    } else if (version == VERSION_MOSS_IMAGE && !moss_uncond_skip.enabled) {
+        LOG_INFO("MOSS uncond skip disabled");
+    }
+    int moss_skipped_steps = 0;
+    // Step caches (EasyCache / UCache / CacheDIT) key the negative prediction by &uncond only, so a windowed skip
+    // drops that entry whenever the effective skip set flips; -1 = no negative pass yet. Never touched by default.
+    int moss_prev_uncond_skip   = -1;
+    int moss_cache_invalidations = 0;
+    // Spectrum extrapolates the guided output from its own history, which that drop never reaches: its history is
+    // cleared at each window edge instead. -1 = no step seen yet.
+    int moss_spectrum_window = -1;
+    int moss_spectrum_resets = 0;
 
     std::vector<float> guidance_schedule = sd::guidance::parse_guidance_schedule(extra_sample_args);
     if (!guidance_schedule.empty() && guidance_schedule.size() != sigmas.size() - 1) {
@@ -2294,7 +2573,8 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
         noise *= eta;
     }
 
-    int64_t last_progress_us     = ggml_time_us();
+    int64_t last_progress_us = ggml_time_us();
+    sd_begin_preview_pass(static_cast<int>(steps));
     SamplePreviewContext preview = prepare_sample_preview_context();
 
     sd::Tensor<float> processed_init_latent       = denoiser->process_latent_in(init_latent);
@@ -2350,11 +2630,29 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
         if (sd_version_is_hunyuan_video(version) && step + 1 < sigmas.size()) {
             hunyuan_timestep_r_tensor = sd::Tensor<float>::from_vector({sigmas[step + 1]});
         }
+        sd::Tensor<float> flux3_target_timestep;
+        if (version == VERSION_FLUX3) {
+            // The sign marks intermediate evaluations; the magnitude is the one-based step.
+            const size_t target_index = static_cast<size_t>(std::abs(step));
+            GGML_ASSERT(target_index > 0 && target_index < sigmas.size());
+            flux3_target_timestep = sd::Tensor<float>::from_vector({sigmas[target_index]});
+        }
         sd::Tensor<float> noised_input = x * c_in;
         if (apply_denoise_mask && (version == VERSION_WAN2_2_TI2V || sd_version_is_ltxav(version) || sd_version_is_lingbot_video(version))) {
             noised_input = noised_input * denoise_mask + sampling_init_latent * (1.0f - denoise_mask);
         }
 
+        if (cache_runtime.spectrum_enabled && version == VERSION_MOSS_IMAGE && moss_uncond_skip.enabled &&
+            moss_uncond_skip.windowed() && !uncond.empty()) {
+            const int moss_window_now = static_cast<int>(sigma >= moss_skip_sigma_low && sigma <= moss_skip_sigma_high);
+            if (moss_spectrum_window >= 0 && moss_spectrum_window != moss_window_now) {
+                cache_runtime.spectrum.reset_history();
+                ++moss_spectrum_resets;
+                LOG_VERBOSE("MOSS uncond skip set changed at step %d: Spectrum history dropped\n", step);
+            }
+            moss_spectrum_window = moss_window_now;
+        }
+
         if (cache_runtime.spectrum_enabled && cache_runtime.spectrum.should_predict()) {
             cache_runtime.spectrum.predict(&denoised);
             if (apply_denoise_mask) {
@@ -2392,6 +2690,7 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
                                 timesteps_tensor,
                                 cond,
                                 &controls);
+        bool uncond_controls_ready = false;
 
         static const std::vector<sd::Tensor<float>> empty_ref_latents;
         bool uncond_without_ref_latents = !img_uncond.empty() &&
@@ -2420,11 +2719,13 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
                     unet_extra.ip_scale   = ip_adapter_strength;
                 }
                 diffusion_params.extra = unet_extra;
-            } else if (sd_version_is_sd3(version)) {
+            } else if (sd_version_is_sd3(version) || version == VERSION_MOSS_IMAGE) {
                 diffusion_params.extra = SkipLayerDiffusionExtra{local_skip_layers};
             } else if (sd_version_is_flux(version) || sd_version_is_flux2(version) || sd_version_is_longcat(version) || sd_version_is_sefi_image(version)) {
                 diffusion_params.extra = FluxDiffusionExtra{&guidance_tensor,
                                                             local_skip_layers};
+            } else if (sd_version_is_flux3(version)) {
+                diffusion_params.extra = Flux3DiffusionExtra{flux3_target_timestep.empty() ? nullptr : &flux3_target_timestep};
             } else if (version == VERSION_QWEN_IMAGE_2_1) {
                 diffusion_params.extra = QwenImage21DiffusionExtra{&condition.c_token_types};
             } else if (sd_version_is_anima(version)) {
@@ -2447,6 +2748,9 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
                     condition.c_token_types.empty() ? nullptr : &condition.c_token_types,
                     condition.c_vinput_mask.empty() ? nullptr : &condition.c_vinput_mask,
                     condition.c_image_embeds.empty() ? nullptr : &condition.c_image_embeds};
+            } else if (version == VERSION_MING_IMAGE) {
+                diffusion_params.extra = MingImageDiffusionExtra{
+                    condition.extra_c_crossattns.empty() ? nullptr : &condition.extra_c_crossattns[0]};
             } else if (sd_version_is_llada_image(version)) {
                 diffusion_params.extra = LLaDAImageDiffusionExtra{
                     condition.extra_c_crossattns.empty() ? nullptr : &condition.extra_c_crossattns[0]};
@@ -2477,14 +2781,45 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
             }
 
             sd::Tensor<float> cached_output;
+            const bool moss_trace_uncond = version == VERSION_MOSS_IMAGE && moss_uncond_skip.enabled &&
+                                           moss_uncond_skip.windowed() && &condition == &uncond;
             if (step_cache.before_condition(&condition, noised_input, &cached_output)) {
+                if (moss_trace_uncond) {
+                    LOG_VERBOSE("MOSS uncond served from step cache at step %d\n", step);
+                }
                 return std::move(cached_output);
             }
+            if (moss_trace_uncond) {
+                LOG_VERBOSE("MOSS uncond computed at step %d\n", step);
+            }
+
+            // A re-enabled condition can miss the cache even when the positive pass was reused.
+            if (!uncond_controls_ready && !uncond.empty() &&
+                (&condition == &uncond || &condition == &img_uncond)) {
+                compute_sample_controls(control_image,
+                                        noised_input,
+                                        timesteps_tensor,
+                                        uncond,
+                                        &controls);
+                uncond_controls_ready = true;
+            }
 
             for (const auto& extension : generation_extensions) {
                 extension->before_diffusion(diffusion_params, step);
             }
 
+            if (cache_qwen_prefix) {
+                auto* extra = std::get_if<QwenImage21DiffusionExtra>(&diffusion_params.extra);
+                if (extra != nullptr) {
+                    auto key         = std::make_tuple(diffusion_params.context, extra->image_slots,
+                                               diffusion_params.ref_image_params.pass_to_dit ? diffusion_params.ref_latents : nullptr);
+                    auto entry       = std::find(qwen_prefix_inputs.begin(), qwen_prefix_inputs.end(), key);
+                    extra->prefix_id = static_cast<uint64_t>(entry - qwen_prefix_inputs.begin()) + 1;
+                    if (entry == qwen_prefix_inputs.end()) {
+                        qwen_prefix_inputs.push_back(key);
+                    }
+                }
+            }
             auto output_opt = work_diffusion_model->compute(n_threads, diffusion_params);
             if (output_opt.empty()) {
                 LOG_ERROR("diffusion model compute failed");
@@ -2495,6 +2830,57 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
             return output_opt;
         };
 
+        auto run_batched_condition = [&](const SDCondition& condition,
+                                         const sd::Tensor<float>* c_concat_override) -> sd::Tensor<float> {
+            const sd::Tensor<float>& condition_concat =
+                c_concat_override != nullptr ? *c_concat_override : condition.c_concat;
+
+            sd::Tensor<float> batched_context = batch_two_condition_tensors(condition.c_crossattn, uncond.c_crossattn);
+            sd::Tensor<float> batched_y       = batch_two_condition_tensors(condition.c_vector, uncond.c_vector);
+            sd::Tensor<float> batched_concat  = batch_two_condition_tensors(condition_concat, uncond.c_concat);
+            if (!condition.c_crossattn.empty() && batched_context.empty()) {
+                return {};
+            }
+            if ((!condition.c_vector.empty() || !uncond.c_vector.empty()) && batched_y.empty()) {
+                return {};
+            }
+            if ((!condition_concat.empty() || !uncond.c_concat.empty()) && batched_concat.empty()) {
+                return {};
+            }
+
+            std::vector<sd::Tensor<float>> uncond_controls;
+            compute_sample_controls(control_image, noised_input, timesteps_tensor, uncond, &uncond_controls);
+            if (controls.size() != uncond_controls.size()) {
+                return {};
+            }
+            std::vector<sd::Tensor<float>> batched_controls;
+            batched_controls.reserve(controls.size());
+            for (size_t i = 0; i < controls.size(); i++) {
+                sd::Tensor<float> batched_control = batch_two_condition_tensors(controls[i], uncond_controls[i]);
+                if (batched_control.empty()) {
+                    return {};
+                }
+                batched_controls.push_back(std::move(batched_control));
+            }
+
+            sd::Tensor<float> batched_x =
+                sd::ops::concat(noised_input, noised_input, static_cast<size_t>(noised_input.dim() - 1));
+
+            DiffusionParams batched_params = diffusion_params;
+            batched_params.x               = &batched_x;
+            batched_params.context         = batched_context.empty() ? nullptr : &batched_context;
+            batched_params.c_concat        = batched_concat.empty() ? nullptr : &batched_concat;
+            batched_params.y               = batched_y.empty() ? nullptr : &batched_y;
+            batched_params.ref_latents     = nullptr;
+            batched_params.extra           = UNetDiffusionExtra{1, &batched_controls, control_strength};
+
+            sd::Tensor<float> output = work_diffusion_model->compute(n_threads, batched_params);
+            if (output.empty()) {
+                LOG_ERROR("batched diffusion model compute failed");
+            }
+            return output;
+        };
+
         const SDCondition* positive_condition      = &cond;
         const sd::Tensor<float>* c_concat_override = nullptr;
         for (const auto& extension : generation_extensions) {
@@ -2508,41 +2894,114 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
             }
         }
 
-        cond_out = run_condition(*positive_condition, c_concat_override);
-        if (cond_out.empty()) {
-            return {};
+        float effective_guidance_scale = guidance_schedule.empty()
+                                             ? cfg_scale
+                                             : guidance_schedule[guidance_schedule.size() - 1 - step];
+
+        float image_guidance_scale = img_cfg_scale;
+
+        constexpr float kEpsilon = 1e-5f;
+
+        bool skip_uncond = false;
+        if (!uncond.empty() && !needs_uncond_denoised && !use_apg_guidance) {
+            if (!img_uncond.empty()) {
+                skip_uncond = std::abs(image_guidance_scale - effective_guidance_scale) < kEpsilon;
+            } else {
+                skip_uncond = std::abs(effective_guidance_scale - 1.0f) < kEpsilon;
+            }
         }
 
-        if (!uncond.empty()) {
-            if (!step_cache.is_step_skipped()) {
-                compute_sample_controls(control_image,
-                                        noised_input,
-                                        timesteps_tensor,
-                                        uncond,
-                                        &controls);
+        bool skip_img_uncond = false;
+        if (!img_uncond.empty() && !needs_uncond_denoised && !use_apg_guidance) {
+            if (!uncond.empty()) {
+                skip_img_uncond = std::abs(image_guidance_scale - 1.0f) < kEpsilon;
+            } else {
+                skip_img_uncond = std::abs(effective_guidance_scale - 1.0f) < kEpsilon;
             }
-            const std::vector<int>* uncond_skip_layers = nullptr;
-            if (is_skiplayer_step && slg_uncond) {
-                LOG_VERBOSE("Skipping layers at uncond step %d\n", step);
-                uncond_skip_layers = &skip_layer_guidance.layers();
-            }
-            uncond_out = run_condition(uncond,
-                                       uncond.c_concat.empty() ? nullptr : &uncond.c_concat,
-                                       uncond_skip_layers,
-                                       nullptr,
-                                       true);
-            if (uncond_out.empty()) {
+        }
+
+        const bool batch_cfg_ok = config_->params.batched_cfg &&
+                                  sd_version_is_unet(version) &&
+                                  !uncond.empty() &&
+                                  img_uncond.empty() &&
+                                  !skip_uncond &&
+                                  !cache_runtime.ucache_enabled() &&
+                                  !(is_skiplayer_step && slg_uncond) &&
+                                  ip_adapter_tokens.empty() &&
+                                  ip_adapter_uncond_tokens.empty() &&
+                                  !config_->animatediff_loaded &&
+                                  (noised_input.dim() < 4 || noised_input.shape()[3] <= 1) &&
+                                  std::none_of(generation_extensions.begin(),
+                                               generation_extensions.end(),
+                                               [](const std::shared_ptr<GenerationExtension>& extension) {
+                                                   return extension->is_enabled();
+                                               });
+
+        if (batch_cfg_ok) {
+            sd::Tensor<float> batched_out = run_batched_condition(*positive_condition, c_concat_override);
+            if (!batched_out.empty() && batched_out.dim() >= 4 && batched_out.shape()[3] == 2) {
+                auto parts = sd::ops::chunk(batched_out, 2, 3);
+                cond_out   = std::move(parts[0]);
+                uncond_out = std::move(parts[1]);
+            }
+        }
+
+        if (cond_out.empty()) {
+            cond_out = run_condition(*positive_condition, c_concat_override);
+            if (cond_out.empty()) {
                 return {};
             }
         }
-        if (!img_uncond.empty()) {
-            img_uncond_out = run_condition(img_uncond,
-                                           img_uncond.c_concat.empty() ? nullptr : &img_uncond.c_concat,
+
+        if (uncond_out.empty() && !uncond.empty()) {
+            if (!skip_uncond) {
+                const std::vector<int>* uncond_skip_layers = nullptr;
+                const std::vector<int> moss_skip_layers    = {22};
+                if (is_skiplayer_step && slg_uncond) {
+                    LOG_VERBOSE("Skipping layers at uncond step %d\n", step);
+                    uncond_skip_layers = &skip_layer_guidance.layers();
+                } else if (version == VERSION_MOSS_IMAGE && moss_uncond_skip.enabled) {
+                    const bool moss_skip_now = !moss_uncond_skip.windowed() ||
+                                               (sigma >= moss_skip_sigma_low && sigma <= moss_skip_sigma_high);
+                    if (moss_uncond_skip.windowed()) {
+                        if (moss_prev_uncond_skip >= 0 && moss_prev_uncond_skip != static_cast<int>(moss_skip_now)) {
+                            step_cache.invalidate_condition(&uncond);
+                            ++moss_cache_invalidations;
+                            LOG_VERBOSE("MOSS uncond skip set changed at step %d: negative-branch step cache dropped\n", step);
+                        }
+                        moss_prev_uncond_skip = static_cast<int>(moss_skip_now);
+                    }
+                    if (moss_skip_now) {
+                        LOG_VERBOSE("MOSS uncond skip layers at step %d (sigma %.4f)\n", step, sigma);
+                        uncond_skip_layers = &moss_skip_layers;
+                        ++moss_skipped_steps;
+                    }
+                }
+                uncond_out = run_condition(uncond,
+                                           uncond.c_concat.empty() ? nullptr : &uncond.c_concat,
+                                           uncond_skip_layers,
                                            nullptr,
-                                           uncond_without_ref_latents ? &empty_ref_latents : nullptr,
                                            true);
-            if (img_uncond_out.empty()) {
-                return {};
+                if (uncond_out.empty()) {
+                    return {};
+                }
+            } else {
+                step_cache.invalidate_condition(&uncond);
+            }
+        }
+
+        if (!img_uncond.empty()) {
+            if (!skip_img_uncond) {
+                img_uncond_out = run_condition(img_uncond,
+                                               img_uncond.c_concat.empty() ? nullptr : &img_uncond.c_concat,
+                                               nullptr,
+                                               uncond_without_ref_latents ? &empty_ref_latents : nullptr,
+                                               true);
+                if (img_uncond_out.empty()) {
+                    return {};
+                }
+            } else {
+                step_cache.invalidate_condition(&img_uncond);
             }
         }
         sd::guidance::GuidanceInput guidance_input;
@@ -2552,7 +3011,7 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
         guidance_input.pred_uncond     = uncond_out.empty() ? nullptr : &uncond_out;
         guidance_input.pred_img_uncond = img_uncond_out.empty() ? nullptr : &img_uncond_out;
 
-        sd::guidance::GuiderOutput guided = guidance_schedule.empty() ? primary_guidance.forward(guidance_input, {}) : primary_guidance.forward(guidance_input, {}, guidance_schedule[guidance_schedule.size() - 1 - step]);
+        sd::guidance::GuiderOutput guided = primary_guidance.forward(guidance_input, {}, effective_guidance_scale);
         if (guided.pred.empty()) {
             return {};
         }
@@ -2607,6 +3066,15 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
 
     auto x0 = std::move(x0_opt);
     sd_sample::log_sample_cache_summary(cache_runtime, steps);
+    if (version == VERSION_MOSS_IMAGE) {
+        LOG_INFO("MOSS uncond skip: %d of %zu steps skipped block 22 on the negative branch", moss_skipped_steps, steps);
+        if (moss_cache_invalidations > 0) {
+            LOG_INFO("MOSS uncond skip: %d window edge(s) dropped the cached negative prediction", moss_cache_invalidations);
+        }
+        if (moss_spectrum_resets > 0) {
+            LOG_INFO("MOSS uncond skip: %d window edge(s) reset the Spectrum history", moss_spectrum_resets);
+        }
+    }
     if (inverse_noise_scaling) {
         x0 = denoiser->inverse_noise_scaling(sigmas[sigmas.size() - 1], x0);
     }
@@ -2630,7 +3098,9 @@ int StableDiffusionGGML::get_diffusion_model_down_factor() {
     if (sd_version_is_dit(version)) {
         if (sd_version_is_sensenova_u1(version)) {
             down_factor = 32;
-        } else if (version == VERSION_QWEN_IMAGE_2_1 || sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_minimax_h3(version)) {
+        } else if (sd_version_is_z_image_l2p(version) || version == VERSION_IRIS) {
+            down_factor = 16;
+        } else if (version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_MING_IMAGE || sd_version_is_wan(version) || sd_version_is_lingbot_video(version) || sd_version_is_minimax_h3(version) || sd_version_is_pixart(version)) {
             down_factor = 2;
         } else {
             down_factor = 1;
@@ -2660,7 +3130,9 @@ int StableDiffusionGGML::get_latent_channel() {
             latent_channel = 3;
         } else if (sd_version_is_sensenova_u1(version)) {
             latent_channel = 3;
-        } else if (sd_version_is_pid(version)) {
+        } else if (sd_version_is_z_image_l2p(version)) {
+            latent_channel = 3;
+        } else if (sd_version_is_pid(version) || version == VERSION_IRIS) {
             latent_channel = 3;
         } else if (sd_version_is_sefi_image(version)) {
             latent_channel = 144;
@@ -2668,6 +3140,8 @@ int StableDiffusionGGML::get_latent_channel() {
             latent_channel = 128;
         } else if (sd_version_is_mage_flow(version)) {
             latent_channel = 128;
+        } else if (sd_version_is_pixart(version)) {
+            latent_channel = 4;
         } else {
             latent_channel = 16;
         }
@@ -2676,7 +3150,7 @@ int StableDiffusionGGML::get_latent_channel() {
 }
 
 int StableDiffusionGGML::get_image_channels() const {
-    return version == VERSION_QWEN_IMAGE_LAYERED || version == VERSION_QWEN_IMAGE_2_1 ? 4 : 3;
+    return version == VERSION_QWEN_IMAGE_LAYERED || version == VERSION_QWEN_IMAGE_2_1 || version == VERSION_MING_IMAGE ? 4 : 3;
 }
 
 int StableDiffusionGGML::get_image_seq_len(int h, int w) {
@@ -2738,8 +3212,63 @@ int StableDiffusionGGML::align_video_frames(int frames) {
     return latent_frames_to_video_frames(video_frames_to_latent_frames(frames));
 }
 
+// After an OOM (last_w/last_h > 0) cap below the failed size: get_tile_sizes may widen a halved request back to it.
+static bool sd_vae_tile_sizes(VAE& vae,
+                              const sd_tiling_params_t& tiling_params,
+                              int64_t latent_w,
+                              int64_t latent_h,
+                              bool circular_x,
+                              bool circular_y,
+                              int last_w,
+                              int last_h,
+                              int& tile_size_w,
+                              int& tile_size_h) {
+    vae.retry_tile_limit_w = last_w > 0 ? std::max(4, last_w - 1) : 0;
+    vae.retry_tile_limit_h = last_h > 0 ? std::max(4, last_h - 1) : 0;
+    tile_size_w            = static_cast<int>(latent_w);
+    tile_size_h            = static_cast<int>(latent_h);
+    float tile_overlap;
+    if (tiling_params.enabled &&
+        !vae.get_tile_sizes(tile_size_w, tile_size_h, tile_overlap, tiling_params,
+                            latent_w, latent_h, circular_x, circular_y)) {
+        return false;
+    }
+    return last_w <= 0 || !tiling_params.enabled || tile_size_w < last_w || tile_size_h < last_h;
+}
+
+struct VAERetryTileLimitGuard {
+    VAE& vae;
+    ~VAERetryTileLimitGuard() {
+        vae.retry_tile_limit_w = 0;
+        vae.retry_tile_limit_h = 0;
+    }
+};
+
 sd::Tensor<float> StableDiffusionGGML::encode_to_vae_latents(const sd::Tensor<float>& x) {
-    auto latents = first_stage_model->encode(n_threads, x, vae_tiling_params, circular_x, circular_y);
+    // Same out-of-memory fallback as decode_first_stage, on a copy of the tiling parameters, so the
+    // smaller tiles apply to this call only.
+    auto tiling_params      = first_stage_model->resolve_tiling_params(vae_tiling_params);
+    const int scale_factor  = first_stage_model->get_scale_factor();
+    const int64_t latent_w  = x.shape()[0] / scale_factor;
+    const int64_t latent_h  = x.shape()[1] / scale_factor;
+    sd::Tensor<float> latents;
+    VAERetryTileLimitGuard limit_guard{*first_stage_model};
+    for (int last_w = 0, last_h = 0;;) {
+        int tile_size_w;
+        int tile_size_h;
+        if (!sd_vae_tile_sizes(*first_stage_model, tiling_params, latent_w, latent_h, circular_x, circular_y,
+                               last_w, last_h, tile_size_w, tile_size_h)) {
+            return {};
+        }
+        latents = first_stage_model->encode(n_threads, x, tiling_params, circular_x, circular_y);
+        if (!latents.empty() ||
+            !sd::backend_fit::prepare_vae_retry_tiling(tiling_params, false, first_stage_model->last_compute_status(),
+                                                       tile_size_w, tile_size_h, scale_factor, true)) {
+            break;
+        }
+        last_w = tile_size_w;
+        last_h = tile_size_h;
+    }
     if (latents.empty()) {
         return {};
     }
@@ -2759,17 +3288,32 @@ sd::Tensor<float> StableDiffusionGGML::encode_first_stage(const sd::Tensor<float
 }
 
 sd::Tensor<float> StableDiffusionGGML::decode_first_stage(const sd::Tensor<float>& x, bool decode_video) {
-    if (sd_version_is_pid(version) || sd_version_is_minit2i(version)) {
+    if (sd_version_is_pid(version) || sd_version_is_minit2i(version) || version == VERSION_IRIS) {
         return sd::ops::clamp((x + 1.f) * 0.5f, 0.0f, 1.0f);
     }
     auto latents                      = first_stage_model->diffusion_to_vae_latents(x);
-    auto decoded                      = first_stage_model->decode(n_threads, latents, vae_tiling_params, decode_video, circular_x, circular_y);
-    const bool prefer_temporal_tiling = decode_video && first_stage_model->can_temporal_tile_decode();
-    while (decoded.empty() &&
-           sd::backend_fit::prepare_vae_decode_retry_tiling(vae_tiling_params, prefer_temporal_tiling)) {
-        decoded = first_stage_model->decode(n_threads, latents, vae_tiling_params, decode_video, circular_x, circular_y);
+    auto tiling_params                = first_stage_model->resolve_tiling_params(vae_tiling_params);
+    const bool prefer_temporal_tiling = decode_video && latents.dim() == 5 && latents.shape()[2] > 1 &&
+                                        first_stage_model->can_temporal_tile_decode();
+    VAERetryTileLimitGuard limit_guard{*first_stage_model};
+    for (int last_w = 0, last_h = 0;;) {
+        int tile_size_w;
+        int tile_size_h;
+        if (!sd_vae_tile_sizes(*first_stage_model, tiling_params, latents.shape()[0], latents.shape()[1],
+                               circular_x, circular_y, last_w, last_h, tile_size_w, tile_size_h)) {
+            return {};
+        }
+        auto decoded = first_stage_model->decode(n_threads, latents, tiling_params, decode_video, circular_x, circular_y);
+        const bool temporal_before = tiling_params.temporal_tiling;
+        if (!decoded.empty() ||
+            !sd::backend_fit::prepare_vae_decode_retry_tiling(tiling_params, prefer_temporal_tiling,
+                                                              first_stage_model->last_compute_status(),
+                                                              tile_size_w, tile_size_h, first_stage_model->get_scale_factor())) {
+            return decoded;
+        }
+        last_w = tiling_params.temporal_tiling != temporal_before ? 0 : tile_size_w;
+        last_h = tiling_params.temporal_tiling != temporal_before ? 0 : tile_size_h;
     }
-    return decoded;
 }
 
 sd::Tensor<float> StableDiffusionGGML::normalize_ltx_video_latents(const sd::Tensor<float>& x) {
@@ -2819,6 +3363,8 @@ std::string StableDiffusionGGML::get_default_ref_image_preset(SDVersion version)
         return "longcat";
     } else if (sd_version_is_flux(version)) {
         return "flux_kontext";
+    } else if (sd_version_is_flux3(version)) {
+        return "flux2";
     } else if (sd_version_is_flux2(version) || sd_version_is_sefi_image(version)) {
         return "flux2";
     } else if (version == VERSION_QWEN_IMAGE_LAYERED) {
diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h
index f2f8a8ced6..c5509d9a9e 100644
--- a/src/pipeline/diffusion_engine.h
+++ b/src/pipeline/diffusion_engine.h
@@ -11,6 +11,7 @@
 #include <optional>
 #include <set>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #include "core/ggml_extend_backend.h"
@@ -26,6 +27,7 @@ class RNG;
 struct Denoiser;
 struct LoraModel;
 struct ConditionerParams;
+class ConditioningCache;
 struct SDCondition;
 struct RefImageParams;
 namespace Wav2Vec2 {
@@ -47,6 +49,11 @@ static inline bool sd_version_supports_ref_latent_img_cfg(SDVersion version) {
 class StableDiffusionGGML {
 public:
     SDBackendManager backend_manager;
+    std::unordered_map<ggml_backend_t, bool> flash_attn_by_backend;
+    SDBackendAssignment flash_attn_assignment;
+    SDBackendAssignment conv_direct_assignment;
+    std::unordered_map<ggml_backend_t, bool> conv2d_direct_by_backend;
+    std::unordered_map<ggml_backend_t, bool> conv3d_direct_by_backend;
 
     SDVersion version;
     bool external_vae_is_invalid = false;
@@ -130,7 +137,7 @@ class StableDiffusionGGML {
                                 &sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path,
                                 &sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path,
                                 &sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path,
-                                &sd_ctx_params_t::tokenizer,
+                                &sd_ctx_params_t::tokenizer, &sd_ctx_params_t::flash_attn_mode, &sd_ctx_params_t::conv_direct_mode,
                                 &sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path,
                                 &sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path,
                                 &sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path,
@@ -178,6 +185,8 @@ class StableDiffusionGGML {
     std::recursive_mutex execution_mutex;
     std::unique_ptr<ModelConfig> config_;
     RunnerState runner_state_;
+    std::unique_ptr<ConditioningCache> conditioning_cache_;
+    std::vector<ModelManager::LoraSpec> conditioning_loras_;
     bool executing_ = false;
 
     std::shared_ptr<Denoiser> denoiser;
@@ -331,6 +340,8 @@ class StableDiffusionGGML {
     bool build_extension_runners();
 
     bool validate_and_load_runners();
+    bool flash_attention_for(SDBackendModule module);
+    bool conv_direct_for(SDBackendModule module, bool conv3d);
 
     bool build_denoiser();
 
@@ -361,6 +372,8 @@ class StableDiffusionGGML {
 
     bool apply_loras(const sd_lora_t* loras, uint32_t lora_count);
 
+    SDCondition get_learned_condition(const ConditionerParams& params);
+
     void reset_generation_extensions();
 
     void prepare_generation_extensions(const sd_pm_params_t& pm_params,
diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp
index a48d04d13d..7de85b3595 100644
--- a/src/pipeline/image.cpp
+++ b/src/pipeline/image.cpp
@@ -10,6 +10,7 @@
 #include "model/vae/vae.hpp"
 #include "request.h"
 #include "runtime/denoiser.hpp"
+#include "runtime/image_preprocess.h"
 #include "upscaler.h"
 
 namespace sd::pipeline {
@@ -34,19 +35,23 @@ namespace sd::pipeline {
             return original_axes;
         }
 
-        int tile_size_x, tile_size_y;
+        int tile_size_w, tile_size_h;
         float overlap;
-        int latent_size_x = request.width / request.vae_scale_factor;
-        int latent_size_y = request.height / request.vae_scale_factor;
-        sd->first_stage_model->get_tile_sizes(tile_size_x,
-                                              tile_size_y,
-                                              overlap,
-                                              sd_img_gen_params->vae_tiling_params,
-                                              latent_size_x,
-                                              latent_size_y);
-
-        sd->circular_x = sd->circular_x && (tile_size_x >= latent_size_x);
-        sd->circular_y = sd->circular_y && (tile_size_y >= latent_size_y);
+        int latent_size_w = request.width / request.vae_scale_factor;
+        int latent_size_h = request.height / request.vae_scale_factor;
+        if (!sd->first_stage_model->get_tile_sizes(tile_size_w,
+                                                   tile_size_h,
+                                                   overlap,
+                                                   sd_img_gen_params->vae_tiling_params,
+                                                   latent_size_w,
+                                                   latent_size_h,
+                                                   sd->circular_x,
+                                                   sd->circular_y)) {
+            return original_axes;
+        }
+
+        sd->circular_x = sd->circular_x && (tile_size_w >= latent_size_w);
+        sd->circular_y = sd->circular_y && (tile_size_h >= latent_size_h);
 
         if (sd->first_stage_model) {
             sd->first_stage_model->set_circular_axes(sd->circular_x, sd->circular_y);
@@ -55,8 +60,8 @@ namespace sd::pipeline {
             sd->preview_vae->set_circular_axes(sd->circular_x, sd->circular_y);
         }
 
-        sd->circular_x = original_axes.circular_x && (tile_size_x < latent_size_x);
-        sd->circular_y = original_axes.circular_y && (tile_size_y < latent_size_y);
+        sd->circular_x = original_axes.circular_x && (tile_size_w < latent_size_w);
+        sd->circular_y = original_axes.circular_y && (tile_size_h < latent_size_h);
 
         return original_axes;
     }
@@ -440,8 +445,7 @@ namespace sd::pipeline {
         sd->compute_ip_adapter_tokens(sd_img_gen_params->ip_adapter_image, sd_img_gen_params->ip_adapter_strength);
         int64_t prepare_start_ms         = ggml_time_ms();
         condition_params.zero_out_masked = false;
-        auto cond                        = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                                       condition_params);
+        auto cond                        = sd->get_learned_condition(condition_params);
         if (cond.empty()) {
             LOG_ERROR("failed to encode prompt");
             return std::nullopt;
@@ -463,6 +467,11 @@ namespace sd::pipeline {
                 // states with a zeroed prompt mask, so no extra text encode is needed.
                 uncond.c_crossattn = cond.c_crossattn;
                 uncond.c_vector    = sd::Tensor<float>::zeros_like(cond.c_vector);
+            } else if (sd->version == VERSION_MING_IMAGE) {
+                uncond.c_crossattn = sd::Tensor<float>::zeros_like(cond.c_crossattn);
+                for (const auto& extra : cond.extra_c_crossattns) {
+                    uncond.extra_c_crossattns.push_back(sd::Tensor<float>::zeros_like(extra));
+                }
             } else if (sd_version_is_sensenova_u1(sd->version)) {
                 auto* sensenova_conditioner = static_cast<SenseNovaU1Conditioner*>(sd->cond_stage_model.get());
                 uncond                      = sensenova_conditioner->get_unconditional_condition(request->negative_prompt);
@@ -479,8 +488,7 @@ namespace sd::pipeline {
                     // LLaDA-Image CFG keeps the source latent but drops its SigVQ features.
                     condition_params.ref_images = nullptr;
                 }
-                uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                     condition_params);
+                uncond = sd->get_learned_condition(condition_params);
                 if (uncond.empty()) {
                     LOG_ERROR("failed to encode negative prompt");
                     return std::nullopt;
@@ -508,8 +516,7 @@ namespace sd::pipeline {
                 if (use_ref_latent_img_cfg) {
                     condition_params.ref_images = &empty_ref_images;
                 }
-                img_uncond = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                         condition_params);
+                img_uncond = sd->get_learned_condition(condition_params);
                 if (img_uncond.empty()) {
                     LOG_ERROR("failed to encode image guidance prompt");
                     return std::nullopt;
@@ -789,15 +796,9 @@ namespace sd::pipeline {
             return false;
         }
 
-        // MiniMax-H3 is video-only. Its denoiser always splits the packed latent into a video and an
-        // audio half, and only generate_video ever computes the audio length, so reaching this
-        // function with an H3 checkpoint is guaranteed to die on
-        // GGML_ASSERT(!audio_input_cache.empty()) with a core dump, after the several minutes it
-        // takes to load the weights, and with nothing in the output pointing at the missing --mode.
-        // (The AnimateDiff path below routes vid_gen back through here, but that is SD1.5 plus a
-        // motion module, never H3.)
-        if (sd_version_is_minimax_h3(sd->version)) {
-            LOG_ERROR("MiniMax-H3 is a video model and cannot be run in img_gen mode; use --mode vid_gen");
+        if (!sd_version_supports_image_generation(sd->version)) {
+            LOG_ERROR("%s cannot be run with generate_image(); use generate_video() or --mode vid_gen in the CLI",
+                      model_version_to_str[sd->version]);
             return false;
         }
 
@@ -806,6 +807,12 @@ namespace sd::pipeline {
         int64_t t0            = ggml_time_ms();
         sd->vae_tiling_params = sd_img_gen_params->vae_tiling_params;
         GenerationRequest request(sd, sd_img_gen_params);
+        sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules);
+        sd_img_gen_params_t processed_params = *sd_img_gen_params;
+        if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
+            return false;
+        sd_img_gen_params = &processed_params;
+        request.pm_params = processed_params.pm_params;
         LOG_INFO("generate_image %dx%d", request.width, request.height);
 
         sd->rng->manual_seed(request.seed);
diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp
index 626bdfee60..4b435170f7 100644
--- a/src/pipeline/model_builders.cpp
+++ b/src/pipeline/model_builders.cpp
@@ -1,5 +1,6 @@
 #include "model_builders.h"
 
+#include <cmath>
 #include <cstring>
 #include <utility>
 
@@ -14,26 +15,32 @@
 #include "model/diffusion/control.hpp"
 #include "model/diffusion/ernie_image.hpp"
 #include "model/diffusion/flux.hpp"
+#include "model/diffusion/flux3.hpp"
 #include "model/diffusion/hidream_o1.hpp"
 #include "model/diffusion/hunyuan.hpp"
 #include "model/diffusion/ideogram4.hpp"
+#include "model/diffusion/iris.hpp"
 #include "model/diffusion/krea2.hpp"
 #include "model/diffusion/lens.hpp"
 #include "model/diffusion/lingbot_video.hpp"
 #include "model/diffusion/llada_image.hpp"
 #include "model/diffusion/ltxv.hpp"
 #include "model/diffusion/mage_flow.hpp"
+#include "model/diffusion/ming_image.hpp"
 #include "model/diffusion/minimax_h3.hpp"
 #include "model/diffusion/minit2i.hpp"
 #include "model/diffusion/mmdit.hpp"
 #include "model/diffusion/model.hpp"
+#include "model/diffusion/moss_image.hpp"
 #include "model/diffusion/pid.hpp"
+#include "model/diffusion/pixart.hpp"
 #include "model/diffusion/qwen_image.hpp"
 #include "model/diffusion/qwen_image_2_1.hpp"
 #include "model/diffusion/sensenova_u1.h"
 #include "model/diffusion/unet.hpp"
 #include "model/diffusion/wan.hpp"
 #include "model/diffusion/z_image.hpp"
+#include "model/diffusion/z_image_l2p.hpp"
 #include "model/vae/auto_encoder_kl.hpp"
 #include "model/vae/hunyuan_vae.hpp"
 #include "model/vae/ltx_audio_vae.hpp"
@@ -168,6 +175,11 @@ namespace sd::model_builders {
                                                                   version,
                                                                   weight_manager,
                                                                   sd_ctx_params->model_args);
+        } else if (sd_version_is_flux3(version)) {
+            result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+                                                               tensor_storage_map, version, "", false, weight_manager, tokenizers);
+            result.diffusion   = std::make_shared<Flux3::Flux3Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+                                                                    tensor_storage_map, "model.diffusion_model", weight_manager);
         } else if (sd_version_is_flux2(version) || sd_version_is_sefi_image(version)) {
             bool is_chroma     = false;
             result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
@@ -291,7 +303,8 @@ namespace sd::model_builders {
                 result.diffusion = std::make_shared<Qwen::QwenImage21Runner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                              tensor_storage_map,
                                                                              "model.diffusion_model",
-                                                                             weight_manager);
+                                                                             weight_manager,
+                                                                             sd_ctx_params->model_args);
             } else {
                 result.diffusion = std::make_shared<Qwen::QwenImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
                                                                            tensor_storage_map,
@@ -300,6 +313,19 @@ namespace sd::model_builders {
                                                                            weight_manager,
                                                                            sd_ctx_params->model_args);
             }
+        } else if (version == VERSION_PIXART) {
+            result.conditioner = std::make_shared<T5CLIPEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+                                                                  tensor_storage_map,
+                                                                  true,
+                                                                  0,
+                                                                  false,
+                                                                  weight_manager,
+                                                                  sd_ctx_params->model_args);
+            result.diffusion   = std::make_shared<PixArt::PixArtRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+                                                                      tensor_storage_map,
+                                                                      "model.diffusion_model",
+                                                                      weight_manager,
+                                                                      sd_ctx_params->model_args);
         } else if (sd_version_is_mage_flow(version)) {
             result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                tensor_storage_map,
@@ -358,6 +384,11 @@ namespace sd::model_builders {
                                                                     tensor_storage_map,
                                                                     "model.diffusion_model",
                                                                     weight_manager);
+        } else if (version == VERSION_MING_IMAGE) {
+            result.conditioner = std::make_shared<MingImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+                                                                     tensor_storage_map, weight_manager, tokenizers);
+            result.diffusion   = std::make_shared<MingImage::MingImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+                                                                            tensor_storage_map, "model.diffusion_model", weight_manager);
         } else if (sd_version_is_z_image(version)) {
             result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                tensor_storage_map,
@@ -371,6 +402,18 @@ namespace sd::model_builders {
                                                                       "model.diffusion_model",
                                                                       version,
                                                                       weight_manager);
+        } else if (sd_version_is_z_image_l2p(version)) {
+            result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+                                                               tensor_storage_map,
+                                                               version,
+                                                               "",
+                                                               false,
+                                                               weight_manager,
+                                                               tokenizers);
+            result.diffusion   = std::make_shared<ZImageL2P::ZImageL2PRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+                                                                            tensor_storage_map,
+                                                                            "model.diffusion_model",
+                                                                            weight_manager);
         } else if (sd_version_is_llada_image(version)) {
             result.conditioner = std::make_shared<LLaDAImageEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
                                                                       tensor_storage_map,
@@ -421,6 +464,16 @@ namespace sd::model_builders {
                                                                   tensor_storage_map,
                                                                   "model.diffusion_model",
                                                                   weight_manager);
+        } else if (version == VERSION_IRIS) {
+            result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+                                                               tensor_storage_map, version, "", false, weight_manager, tokenizers);
+            result.diffusion   = std::make_shared<Iris::IrisRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+                                                                  tensor_storage_map, "model.diffusion_model", weight_manager);
+        } else if (version == VERSION_MOSS_IMAGE) {
+            result.conditioner = std::make_shared<LLMEmbedder>(ctx.backends.runtime_backend(SDBackendModule::TE),
+                                                               tensor_storage_map, version, "", false, weight_manager, tokenizers);
+            result.diffusion   = std::make_shared<MOSSImage::MOSSImageRunner>(ctx.backends.runtime_backend(SDBackendModule::DIFFUSION),
+                                                                            tensor_storage_map, "model.diffusion_model", weight_manager);
         } else {  // SD1.x SD2.x SDXL
             std::map<std::string, std::string> embbeding_map;
             for (uint32_t i = 0; i < sd_ctx_params->embedding_count; i++) {
@@ -437,10 +490,6 @@ namespace sd::model_builders {
                                                                  "model.diffusion_model",
                                                                  version,
                                                                  weight_manager);
-            if (sd_ctx_params->diffusion_conv_direct) {
-                LOG_INFO("Using Conv2d direct in the diffusion model");
-                result.diffusion->set_conv2d_direct_enabled(true);
-            }
         }
 
         if (strlen(SAFE_STR(sd_ctx_params->ip_adapter_path)) > 0 && result.clip_vision == nullptr) {
@@ -492,7 +541,7 @@ namespace sd::model_builders {
         }
 
         auto create_tae = [&](bool decode_only) -> std::shared_ptr<VAE> {
-            if (sd_version_uses_wan_vae(version) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version)) {
+            if ((sd_version_uses_wan_vae(version) && version != VERSION_QWEN_IMAGE_2_1) || sd_version_is_hunyuan_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version)) {
                 return std::make_shared<TinyVideoAutoEncoder>(ctx.backends.runtime_backend(SDBackendModule::VAE),
                                                               tensor_storage_map,
                                                               "decoder",
@@ -561,6 +610,23 @@ namespace sd::model_builders {
                                                              false,
                                                              vae_version,
                                                              weight_manager);
+                if (sd_version_is_pixart(version)) {
+                    // Alpha-512 and Sigma share tensor layouts; Alpha-512 needs an explicit scale override.
+                    if (tensor_storage_map.count("model.diffusion_model.csize_embedder.mlp.0.weight") != 0) {
+                        model->scale_factor = 0.18215f;
+                    }
+                    for (const auto& [key, value] : parse_key_value_args(sd_ctx_params->model_args, "model arg")) {
+                        if (key == "pixart_vae_scale_factor") {
+                            float parsed = 0.f;
+                            if (parse_strict_float(value, parsed) && std::isfinite(parsed) && parsed > 0.f) {
+                                model->scale_factor = parsed;
+                            } else {
+                                LOG_WARN("ignoring invalid PixArt model arg '%s=%s'", key.c_str(), value.c_str());
+                            }
+                        }
+                    }
+                    LOG_VERBOSE("pixart: VAE scale factor = %.5f", model->scale_factor);
+                }
                 if (sd_version_is_sdxl(version) &&
                     (strlen(SAFE_STR(sd_ctx_params->vae_path)) == 0 || sd_ctx_params->force_sdxl_vae_conv_scale || options.external_vae_is_invalid)) {
                     float vae_conv_2d_scale = 1.f / 32.f;
@@ -574,7 +640,12 @@ namespace sd::model_builders {
             }
         };
 
-        if (version == VERSION_CHROMA_RADIANCE || version == VERSION_HIDREAM_O1 || sd_version_is_minit2i(version) || sd_version_is_sensenova_u1(version)) {
+        if (version == VERSION_CHROMA_RADIANCE ||
+            version == VERSION_HIDREAM_O1 ||
+            sd_version_is_minit2i(version) ||
+            sd_version_is_sensenova_u1(version) ||
+            sd_version_is_z_image_l2p(version) ||
+            version == VERSION_IRIS) {
             LOG_INFO("using FakeVAE");
             result.vae = std::make_shared<FakeVAE>(version,
                                                    ctx.backends.runtime_backend(SDBackendModule::VAE),
@@ -605,15 +676,6 @@ namespace sd::model_builders {
             }
         }
 
-        if (sd_ctx_params->vae_conv_direct) {
-            LOG_INFO("Using Conv2d/Conv3d direct in the vae model");
-            result.vae->set_conv2d_direct_enabled(true);
-            result.vae->set_conv3d_direct_enabled(true);
-            if (result.preview) {
-                result.preview->set_conv2d_direct_enabled(true);
-                result.preview->set_conv3d_direct_enabled(true);
-            }
-        }
         if (result.vae) {
             result.vae->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
         }
@@ -640,10 +702,6 @@ namespace sd::model_builders {
                                                         version,
                                                         "",
                                                         weight_manager);
-        if (sd_ctx_params->diffusion_conv_direct) {
-            LOG_INFO("Using Conv2d direct in the control net");
-            control_net->set_conv2d_direct_enabled(true);
-        }
         control_net->set_scale_overrides(sd_ctx_params->linear_scale, sd_ctx_params->attn_scale);
         runner = std::move(control_net);
         return true;
diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp
index cfc54183af..4256fee4e2 100644
--- a/src/pipeline/request.cpp
+++ b/src/pipeline/request.cpp
@@ -384,17 +384,19 @@ namespace sd::pipeline {
         total_steps = sample_steps + std::max(0, high_noise_sample_steps);
 
         if (sample_params->custom_sigmas_count > 0) {
-            sigmas      = std::vector<float>(sample_params->custom_sigmas,
+            sigmas = std::vector<float>(sample_params->custom_sigmas,
                                         sample_params->custom_sigmas + sample_params->custom_sigmas_count);
-            total_steps = static_cast<int>(sigmas.size()) - 1;
-            LOG_WARN("total_steps != custom_sigmas_count - 1, set total_steps to %d", total_steps);
-            if (sample_steps >= total_steps) {
-                sample_steps = total_steps;
-                LOG_WARN("total_steps != custom_sigmas_count - 1, set sample_steps to %d", sample_steps);
-            }
-            if (high_noise_sample_steps > 0) {
-                high_noise_sample_steps = total_steps - sample_steps;
-                LOG_WARN("total_steps != custom_sigmas_count - 1, set high_noise_sample_steps to %d", high_noise_sample_steps);
+            if (total_steps != sample_params->custom_sigmas_count - 1) {
+                total_steps = sample_params->custom_sigmas_count - 1;
+                LOG_WARN("total_steps != custom_sigmas_count - 1, set total_steps to %d", total_steps);
+                if ((sample_steps > total_steps) || (high_noise_sample_steps <= 0 && sample_steps != total_steps)) {
+                    sample_steps = total_steps;
+                    LOG_WARN("total_steps != custom_sigmas_count - 1, set sample_steps to %d", sample_steps);
+                }
+                if (high_noise_sample_steps > 0) {
+                    high_noise_sample_steps = total_steps - sample_steps;
+                    LOG_WARN("total_steps != custom_sigmas_count - 1, set high_noise_sample_steps to %d", high_noise_sample_steps);
+                }
             }
         } else {
             scheduler_t scheduler = resolve_scheduler(sd,
diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp
index 503afa530a..2fcff4c9d4 100644
--- a/src/pipeline/video.cpp
+++ b/src/pipeline/video.cpp
@@ -15,6 +15,7 @@
 #include "model/vae/vae.hpp"
 #include "request.h"
 #include "runtime/denoiser.hpp"
+#include "runtime/image_preprocess.h"
 
 namespace sd::pipeline {
 
@@ -470,11 +471,15 @@ namespace sd::pipeline {
         sd::Tensor<float> end_image;
 
         if (sd_vid_gen_params->init_image.data) {
-            start_image = sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height);
+            start_image = ensure_image_tensor_channels(
+                sd_image_to_tensor(sd_vid_gen_params->init_image, request->width, request->height),
+                sd->get_image_channels());
         }
 
         if (sd_vid_gen_params->end_image.data) {
-            end_image = sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height);
+            end_image = ensure_image_tensor_channels(
+                sd_image_to_tensor(sd_vid_gen_params->end_image, request->width, request->height),
+                sd->get_image_channels());
         }
 
         if (sd_version_is_minimax_h3(sd->version)) {
@@ -1157,8 +1162,7 @@ namespace sd::pipeline {
         }
 
         int64_t prepare_start_ms = ggml_time_ms();
-        embeds.cond              = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                               condition_params);
+        embeds.cond              = sd->get_learned_condition(condition_params);
         if (embeds.cond.empty()) {
             LOG_ERROR("failed to encode video prompt");
             return std::nullopt;
@@ -1183,8 +1187,7 @@ namespace sd::pipeline {
         }
         if (request.use_uncond) {
             condition_params.text = request.negative_prompt;
-            embeds.uncond         = sd->cond_stage_model->get_learned_condition(sd->n_threads,
-                                                                                condition_params);
+            embeds.uncond         = sd->get_learned_condition(condition_params);
             if (embeds.uncond.empty()) {
                 LOG_ERROR("failed to encode negative video prompt");
                 return std::nullopt;
@@ -1416,7 +1419,9 @@ namespace sd::pipeline {
         sd::Tensor<float> video_mask = make_ltxav_video_denoise_mask(video_latent, 1.f);
 
         if (sd_vid_gen_params->init_image.data != nullptr) {
-            sd::Tensor<float> start_image = sd_image_to_tensor(sd_vid_gen_params->init_image, image_width, image_height);
+            sd::Tensor<float> start_image = ensure_image_tensor_channels(
+                sd_image_to_tensor(sd_vid_gen_params->init_image, image_width, image_height),
+                sd->get_image_channels());
             if (!apply_ltxav_condition_image_by_latent_index(sd,
                                                              start_image,
                                                              &video_latent,
@@ -1429,7 +1434,9 @@ namespace sd::pipeline {
         }
 
         if (sd_vid_gen_params->end_image.data != nullptr) {
-            sd::Tensor<float> end_image        = sd_image_to_tensor(sd_vid_gen_params->end_image, image_width, image_height);
+            sd::Tensor<float> end_image = ensure_image_tensor_channels(
+                sd_image_to_tensor(sd_vid_gen_params->end_image, image_width, image_height),
+                sd->get_image_channels());
             sd::Tensor<float> end_image_latent = encode_ltxav_condition_image(sd, end_image, "end");
             if (end_image_latent.empty()) {
                 return false;
@@ -1518,6 +1525,7 @@ namespace sd::pipeline {
         img_gen_params.qwen_image_layers = 0;
         img_gen_params.circular_x        = sd_vid_gen_params->circular_x;
         img_gen_params.circular_y        = sd_vid_gen_params->circular_y;
+        img_gen_params.image_preprocess  = sd_vid_gen_params->image_preprocess;
 
         sd->animatediff_num_frames = n_frames;
         bool ok                    = generate_image(sd, &img_gen_params, frames_out, num_frames_out);
@@ -1548,6 +1556,11 @@ namespace sd::pipeline {
         sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
         sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y);
         GenerationRequest request(sd, sd_vid_gen_params);
+        sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules);
+        sd_vid_gen_params_t processed_params = *sd_vid_gen_params;
+        if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
+            return false;
+        sd_vid_gen_params = &processed_params;
         if (fps_out != nullptr) {
             *fps_out = request.fps;
         }
diff --git a/src/runtime/denoiser.hpp b/src/runtime/denoiser.hpp
index 416859e6da..6fbcb43df3 100644
--- a/src/runtime/denoiser.hpp
+++ b/src/runtime/denoiser.hpp
@@ -728,12 +728,20 @@ inline float flux_time_shift(float mu, float sigma, float t) {
 
 // https://github.com/black-forest-labs/flux/blob/main/src/flux/sampling.py#L289
 struct FluxScheduler : SigmaScheduler {
-    int image_seq_len = 0;
-    float base_shift  = 0.5f;
-    float max_shift   = 1.15f;
-
-    explicit FluxScheduler(int image_seq_len, const char* extra_sample_args = nullptr)
+    int image_seq_len      = 0;
+    int base_image_seq_len = 256;
+    int max_image_seq_len  = 4096;
+    float base_shift       = 0.5f;
+    float max_shift        = 1.15f;
+    float shift_terminal   = 0.0f;
+
+    FluxScheduler(int image_seq_len, SDVersion version, const char* extra_sample_args = nullptr)
         : image_seq_len(image_seq_len) {
+        if (version == VERSION_QWEN_IMAGE_2_1) {
+            max_image_seq_len = 8192;
+            max_shift         = 0.9f;
+            shift_terminal    = 0.02f;
+        }
         parse_extra_sample_args(extra_sample_args);
     }
 
@@ -752,10 +760,8 @@ struct FluxScheduler : SigmaScheduler {
     }
 
     float compute_mu() const {
-        constexpr float base_shift_anchor = 256.0f;
-        constexpr float max_shift_anchor  = 4096.0f;
-        float m                           = (max_shift - base_shift) / (max_shift_anchor - base_shift_anchor);
-        float b                           = base_shift - m * base_shift_anchor;
+        float m = (max_shift - base_shift) / static_cast<float>(max_image_seq_len - base_image_seq_len);
+        float b = base_shift - m * static_cast<float>(base_image_seq_len);
         return static_cast<float>(image_seq_len) * m + b;
     }
 
@@ -764,7 +770,7 @@ struct FluxScheduler : SigmaScheduler {
         sigmas.reserve(n + 1);
 
         float mu = compute_mu();
-        LOG_VERBOSE("Flux scheduler: image_seq_len=%d, steps=%u, mu=%.3f", image_seq_len, n, mu);
+        LOG_VERBOSE("Flux scheduler: image_seq_len=%d, steps=%u, mu=%.3f, shift_terminal=%.3f", image_seq_len, n, mu, shift_terminal);
 
         if (n == 0) {
             sigmas.push_back(1.0f);
@@ -780,6 +786,16 @@ struct FluxScheduler : SigmaScheduler {
             }
         }
 
+        if (shift_terminal > 0.0f && n > 1) {
+            // The terminal shift applies to the last model evaluation, not the final zero sigma.
+            float scale_factor = (1.0f - sigmas[n - 1]) / (1.0f - shift_terminal);
+            if (std::isfinite(scale_factor) && scale_factor > 0.0f) {
+                for (uint32_t i = 0; i < n; ++i) {
+                    sigmas[i] = 1.0f - (1.0f - sigmas[i]) / scale_factor;
+                }
+            }
+        }
+
         sigmas[n] = 0.0f;
         return sigmas;
     }
@@ -1178,7 +1194,7 @@ struct Denoiser {
             }
             case FLUX_SCHEDULER: {
                 LOG_INFO("get_sigmas with Flux scheduler");
-                scheduler = std::make_shared<FluxScheduler>(image_seq_len, extra_sample_args);
+                scheduler = std::make_shared<FluxScheduler>(image_seq_len, version, extra_sample_args);
                 break;
             }
             default:
@@ -1353,6 +1369,42 @@ struct DiscreteFlowDenoiser : public Denoiser {
     }
 };
 
+struct MingImageFlowDenoiser : DiscreteFlowDenoiser {
+    float resolution_shift = std::exp(1.35f);
+
+    MingImageFlowDenoiser()
+        : DiscreteFlowDenoiser(INFINITY) {}
+
+    float sigma_min() override { return 0.f; }
+    float sigma_max() override { return 1.f; }
+
+    float t_to_sigma(float t) override {
+        return time_snr_shift(std::isfinite(shift) ? shift : resolution_shift, (t + 1.f) / 1000.f);
+    }
+
+    std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler, SDVersion version, const char* extra_sample_args = nullptr) override {
+        const float tokens = static_cast<float>(image_seq_len) / 4.f;
+        const float mu     = tokens >= 4096.f ? 1.35f : 0.5f + (tokens - 256.f) * (1.15f - 0.5f) / (4096.f - 256.f);
+        resolution_shift   = std::exp(mu);
+        if (scheduler != DISCRETE_SCHEDULER) {
+            return DiscreteFlowDenoiser::get_sigmas(n, image_seq_len, scheduler, version, extra_sample_args);
+        }
+        if (n == 0) {
+            return {};
+        }
+        const float factor = std::isfinite(shift) ? shift : resolution_shift;
+        std::vector<float> sigmas;
+        sigmas.reserve(n + 1);
+        for (uint32_t i = 0; i < n; ++i) {
+            const float t = n == 1 ? 1.f : 1.f - static_cast<float>(i) / static_cast<float>(n - 1);
+            sigmas.push_back(time_snr_shift(factor, t));
+        }
+        // Upstream sets sigma_min to zero before linspace, then appends the terminal zero.
+        sigmas.push_back(0.f);
+        return sigmas;
+    }
+};
+
 struct H3AVFlowDenoiser : public DiscreteFlowDenoiser {
     int64_t video_channels;
     float audio_shift;
@@ -1400,6 +1452,29 @@ struct FluxFlowDenoiser : public DiscreteFlowDenoiser {
     }
 };
 
+struct Flux3FlowDenoiser : public FluxFlowDenoiser {
+    float t_to_sigma(float t) override {
+        float u = (t + 1.f) / TIMESTEPS;
+        return shift * u / (1.f + (shift - 1.f) * u);
+    }
+
+    std::vector<float> get_sigmas(uint32_t n, int image_seq_len, scheduler_t scheduler_type, SDVersion version, const char* extra_sample_args = nullptr) override {
+        if (scheduler_type != DISCRETE_SCHEDULER) {
+            return Denoiser::get_sigmas(n, image_seq_len, scheduler_type, version, extra_sample_args);
+        }
+        if (n == 0) {
+            return {};
+        }
+        std::vector<float> sigmas;
+        sigmas.reserve(n + 1);
+        for (uint32_t i = 0; i <= n; ++i) {
+            float u = 1.f - static_cast<float>(i) / n;
+            sigmas.push_back(shift * u / (1.f + (shift - 1.f) * u));
+        }
+        return sigmas;
+    }
+};
+
 struct SefiFlowDenoiser;
 
 struct SefiFlowDenoiser : public FluxFlowDenoiser {
@@ -1767,7 +1842,10 @@ static sd::Tensor<float> sample_euler(denoise_cb_t model,
                                       const std::vector<float>& sigmas) {
     int steps = static_cast<int>(sigmas.size()) - 1;
     for (int i = 0; i < steps; i++) {
-        float sigma       = sigmas[i];
+        float sigma = sigmas[i];
+        if (sigma == sigmas[i + 1]) {
+            continue;
+        }
         auto denoised_opt = model(x, sigma, i + 1);
         if (denoised_opt.pred.empty()) {
             return {};
diff --git a/src/runtime/guidance.cpp b/src/runtime/guidance.cpp
index bfb773b0a1..0df0f7a772 100644
--- a/src/runtime/guidance.cpp
+++ b/src/runtime/guidance.cpp
@@ -64,6 +64,34 @@ namespace sd::guidance {
         return uncond;
     }
 
+    MossUncondSkipParams parse_moss_uncond_skip_args(const char* extra_sample_args) {
+        MossUncondSkipParams params;
+        for (const auto& [key, value] : parse_key_value_args(extra_sample_args, "extra sample arg")) {
+            if (key == "moss_uncond_skip") {
+                bool parsed = true;
+                if (!parse_strict_bool(value, parsed)) {
+                    LOG_WARN("ignoring invalid MOSS extra sample arg '%s=%s'", key.c_str(), value.c_str());
+                    continue;
+                }
+                params.enabled = parsed;
+            } else if (key == "moss_uncond_skip_start" || key == "moss_uncond_skip_end") {
+                float parsed = 0.0f;
+                if (!parse_strict_float(value, parsed) || !(parsed >= 0.0f && parsed <= 1.0f)) {
+                    LOG_WARN("ignoring invalid MOSS extra sample arg '%s=%s' (expected 0..1)", key.c_str(), value.c_str());
+                    continue;
+                }
+                (key == "moss_uncond_skip_start" ? params.start : params.end) = parsed;
+            }
+        }
+        if (params.start > params.end) {
+            LOG_WARN("moss_uncond_skip_start %.4f > moss_uncond_skip_end %.4f: keeping the skip on every step",
+                     params.start, params.end);
+            params.start = 0.0f;
+            params.end   = 1.0f;
+        }
+        return params;
+    }
+
     std::vector<float> parse_guidance_schedule_from_spec(std::string spec) {
         std::vector<float> schedule;
 
diff --git a/src/runtime/guidance.h b/src/runtime/guidance.h
index 3de337042d..a81b9a2595 100644
--- a/src/runtime/guidance.h
+++ b/src/runtime/guidance.h
@@ -28,6 +28,17 @@ namespace sd::guidance {
     AdaptiveProjectedGuidanceParams parse_adaptive_projected_guidance_args(const char* extra_sample_args);
     bool is_adaptive_projected_guidance_enabled(const AdaptiveProjectedGuidanceParams& params);
     bool parse_skip_layer_guidance_uncond_arg(const char* extra_sample_args);
+
+    // MOSS-Image's built-in negative-branch skip (block 22 on the uncond pass). extra_sample_args
+    // moss_uncond_skip=false turns it off; moss_uncond_skip_start / moss_uncond_skip_end bound it in
+    // sigma-percent of the shifted flow schedule (ComfyUI percent_to_sigma). Defaults keep every step.
+    struct MossUncondSkipParams {
+        bool enabled = true;
+        float start  = 0.0f;
+        float end    = 1.0f;
+        bool windowed() const { return start > 0.0f || end < 1.0f; }
+    };
+    MossUncondSkipParams parse_moss_uncond_skip_args(const char* extra_sample_args);
     std::vector<float> parse_guidance_schedule(const char* extra_sample_args);
 
     struct GuidanceInput {
diff --git a/src/runtime/image_preprocess.cpp b/src/runtime/image_preprocess.cpp
new file mode 100644
index 0000000000..a8df7c3b56
--- /dev/null
+++ b/src/runtime/image_preprocess.cpp
@@ -0,0 +1,447 @@
+#include "image_preprocess.h"
+#include "core/util.h"
+
+#include <climits>
+#include <set>
+
+namespace sd {
+
+    static constexpr std::pair<const char*, ImageTarget> image_targets[] = {
+        {"init", ImageTarget::Init},
+        {"end", ImageTarget::End},
+        {"mask", ImageTarget::Mask},
+        {"control", ImageTarget::Control},
+        {"ref", ImageTarget::Ref},
+        {"ip-adapter", ImageTarget::IPAdapter},
+        {"id", ImageTarget::ID},
+        {"control-frame", ImageTarget::ControlFrame},
+    };
+
+    static constexpr std::pair<const char*, ImageResizeMode> image_resize_modes[] = {
+        {"auto", ImageResizeMode::Auto},
+        {"none", ImageResizeMode::None},
+        {"stretch", ImageResizeMode::Stretch},
+        {"crop", ImageResizeMode::Crop},
+        {"crop-resize", ImageResizeMode::CropResize},
+        {"fit-pad", ImageResizeMode::FitPad},
+    };
+
+    template <typename T, size_t N>
+    static bool parse_enum(const std::string& text, const std::pair<const char*, T> (&names)[N], T& value) {
+        for (const auto& entry : names) {
+            if (text == entry.first) {
+                value = entry.second;
+                return true;
+            }
+        }
+        return false;
+    }
+
+    template <typename T, size_t N>
+    static const char* enum_name(T value, const std::pair<const char*, T> (&names)[N]) {
+        for (const auto& entry : names) {
+            if (value == entry.second)
+                return entry.first;
+        }
+        return "unknown";
+    }
+
+    template <typename T>
+    static bool one_of(T value, std::initializer_list<T> choices) {
+        return std::find(choices.begin(), choices.end(), value) != choices.end();
+    }
+
+    static bool one_of(const std::string& value, std::initializer_list<const char*> choices) {
+        for (const char* choice : choices) {
+            if (value == choice)
+                return true;
+        }
+        return false;
+    }
+
+    static ImageResizeMode resolve_mode(const std::map<std::string, std::string>& options, ImageResizeMode default_mode) {
+        auto it              = options.find("mode");
+        ImageResizeMode mode = ImageResizeMode::Auto;
+        if (it != options.end())
+            parse_enum(it->second, image_resize_modes, mode);
+        if (mode != ImageResizeMode::Auto)
+            return mode;
+        return options.count("width") && default_mode == ImageResizeMode::None ? ImageResizeMode::Stretch : default_mode;
+    }
+
+    bool ImagePreprocessor::fail(const std::string& message) const {
+        LOG_ERROR("image preprocessing: %s", message.c_str());
+        valid_ = false;
+        return false;
+    }
+
+    ImagePreprocessor::ImagePreprocessor(const char* text) {
+        if (text == nullptr || trim(text).empty())
+            return;
+        for (const auto& part : split_string(text, ';')) {
+            ImagePreprocessRule rule;
+            std::set<std::string> keys;
+            if (trim(part).empty()) {
+                fail("empty rule");
+                return;
+            }
+            for (const auto& entry : split_string(part, ',')) {
+                size_t equal = entry.find('=');
+                if (equal == std::string::npos) {
+                    fail("expected key=value: " + entry);
+                    return;
+                }
+                std::string key   = trim(entry.substr(0, equal));
+                std::string value = trim(entry.substr(equal + 1));
+                bool ok           = !value.empty() && keys.insert(key).second;
+                int number        = 0;
+                if (key == "target") {
+                    ok &= parse_enum(value, image_targets, rule.target);
+                } else if (key == "index") {
+                    ok &= parse_strict_int(value, rule.index) && rule.index >= 0;
+                } else {
+                    if (key == "mode") {
+                        ImageResizeMode mode;
+                        ok &= parse_enum(value, image_resize_modes, mode);
+                    } else if (key == "filter") {
+                        ok &= one_of(value, {"auto", "nearest", "nearest-exact", "bilinear", "bicubic", "lanczos"});
+                    } else if (key == "antialias") {
+                        ok &= one_of(value, {"auto", "true", "false"});
+                    } else if (key == "canny") {
+                        ok &= one_of(value, {"true", "false"});
+                    } else if (key == "anchor") {
+                        ok &= one_of(value, {"center", "top", "bottom", "left", "right"});
+                    } else if (key == "width" || key == "height") {
+                        ok &= parse_strict_int(value, number) && number > 0;
+                    } else if (key == "pad_color") {
+                        ok &= value.size() == 7 || value.size() == 9;
+                        ok &= !value.empty() && value[0] == '#';
+                        for (size_t i = 1; i < value.size(); ++i)
+                            ok &= std::isxdigit(static_cast<unsigned char>(value[i])) != 0;
+                    } else {
+                        ok = false;
+                    }
+                    rule.options[key] = value;
+                }
+                if (!ok) {
+                    fail("invalid or duplicate option: " + entry);
+                    return;
+                }
+            }
+            if (!keys.count("target") || rule.options.empty() ||
+                rule.options.count("width") != rule.options.count("height") ||
+                (rule.index >= 0 && !one_of(rule.target, {ImageTarget::Ref, ImageTarget::ID, ImageTarget::ControlFrame}))) {
+                fail("invalid target, index, or incomplete dimensions: " + part);
+                return;
+            }
+            rules_.push_back(std::move(rule));
+        }
+        for (const auto& rule : rules_) {
+            const auto options = resolve_options(rule.target, std::max(0, rule.index));
+            if (options.count("antialias") && options.at("antialias") == "true" && options.count("filter") &&
+                one_of(options.at("filter"), {"nearest", "nearest-exact"})) {
+                fail("antialias requires bilinear, bicubic, or lanczos");
+                return;
+            }
+        }
+    }
+
+    std::map<std::string, std::string> ImagePreprocessor::resolve_options(ImageTarget target, int index) const {
+        std::map<std::string, std::string> options;
+        for (int specificity = 0; specificity < 2; ++specificity) {
+            for (const auto& rule : rules_) {
+                if (rule.target == target &&
+                    rule.index == (specificity == 0 ? -1 : index)) {
+                    for (const auto& entry : rule.options)
+                        options[entry.first] = entry.second;
+                }
+            }
+        }
+        return options;
+    }
+
+    bool ImagePreprocessor::validate_inputs(const sd_img_gen_params_t& params) const {
+        const std::map<ImageTarget, int> counts = {
+            {ImageTarget::Init, params.init_image.data != nullptr},
+            {ImageTarget::Mask, params.mask_image.data != nullptr},
+            {ImageTarget::Control, params.control_image.data != nullptr},
+            {ImageTarget::IPAdapter, params.ip_adapter_image.data != nullptr},
+            {ImageTarget::Ref, params.ref_images != nullptr ? params.ref_images_count : 0},
+            {ImageTarget::ID, params.pm_params.id_images != nullptr ? params.pm_params.id_images_count : 0},
+        };
+        for (const auto& rule : rules_) {
+            auto it   = counts.find(rule.target);
+            int count = it == counts.end() ? 0 : it->second;
+            if (count <= 0 || rule.index >= count) {
+                return fail(std::string("rule targets an unavailable image: ") + enum_name(rule.target, image_targets));
+            }
+        }
+        return valid_;
+    }
+
+    bool ImagePreprocessor::validate_inputs(const sd_vid_gen_params_t& params) const {
+        const std::map<ImageTarget, int> counts = {
+            {ImageTarget::Init, params.init_image.data != nullptr},
+            {ImageTarget::End, params.end_image.data != nullptr},
+            {ImageTarget::Ref, params.ref_images != nullptr ? params.ref_images_count : 0},
+            {ImageTarget::ControlFrame, params.control_frames != nullptr ? params.control_frames_size : 0},
+        };
+        for (const auto& rule : rules_) {
+            auto it   = counts.find(rule.target);
+            int count = it == counts.end() ? 0 : it->second;
+            if (count <= 0 || rule.index >= count)
+                return fail(std::string("rule targets an unavailable video input: ") + enum_name(rule.target, image_targets));
+        }
+        return valid_;
+    }
+
+    static int anchor_offset(int remaining, const std::string& anchor, bool horizontal) {
+        if (anchor == (horizontal ? "left" : "top"))
+            return 0;
+        if (anchor == (horizontal ? "right" : "bottom"))
+            return remaining;
+        return remaining / 2;
+    }
+
+    Tensor<float> ImagePreprocessor::apply_transform(const Tensor<float>& image, const std::map<std::string, std::string>& options, ImageTransform p, const std::string& label, ops::InterpolateMode default_filter) const {
+        auto value = [&](const char* key, const char* fallback) {
+            auto it = options.find(key);
+            return it == options.end() ? std::string(fallback) : it->second;
+        };
+        std::string filter        = value("filter", "auto");
+        ops::InterpolateMode mode = default_filter;
+        if (filter == "nearest")
+            mode = ops::InterpolateMode::Nearest;
+        if (filter == "nearest-exact")
+            mode = ops::InterpolateMode::NearestExact;
+        if (filter == "bilinear")
+            mode = ops::InterpolateMode::Bilinear;
+        if (filter == "bicubic")
+            mode = ops::InterpolateMode::Bicubic;
+        if (filter == "lanczos")
+            mode = ops::InterpolateMode::Lanczos;
+        bool filtered  = ops::is_2d_filter_interpolate_mode(mode);
+        bool antialias = value("antialias", "auto") == "true" ||
+                         (value("antialias", "auto") == "auto" && filtered &&
+                          (p.resize_width < p.crop_width || p.resize_height < p.crop_height));
+        if (antialias && !filtered) {
+            fail(label + ": antialias requires bilinear, bicubic, or lanczos");
+            return {};
+        }
+        auto cropped = ops::slice(ops::slice(image, 0, p.x, p.x + p.crop_width), 1, p.y, p.y + p.crop_height);
+        int channels = static_cast<int>(image.shape()[2]);
+        bool resize  = p.resize_width != p.crop_width || p.resize_height != p.crop_height;
+        if (resize && channels == 4 && filtered) {
+            for (int64_t i = 0, pixels = cropped.shape()[0] * cropped.shape()[1]; i < pixels; ++i) {
+                for (int c = 0; c < 3; ++c)
+                    cropped[i + c * pixels] *= cropped[i + 3 * pixels];
+            }
+        }
+        auto resized = ops::interpolate(cropped, {p.resize_width, p.resize_height, channels, 1}, mode, false, antialias);
+        if (resize && channels == 4 && filtered) {
+            for (int64_t i = 0, pixels = resized.shape()[0] * resized.shape()[1]; i < pixels; ++i) {
+                float alpha = std::clamp(resized[i + 3 * pixels], 0.f, 1.f);
+                for (int c = 0; c < 3; ++c)
+                    resized[i + c * pixels] = alpha > 1e-6f ? resized[i + c * pixels] / alpha : 0.f;
+            }
+        }
+        resized = ops::clamp(resized, 0.f, 1.f);
+        Tensor<float> output({p.width, p.height, channels, 1});
+        std::string color = value("pad_color", "#000000ff");
+        if (color.size() == 7)
+            color += "ff";
+        uint8_t rgba[4];
+        for (int c = 0; c < 4; ++c)
+            rgba[c] = static_cast<uint8_t>(std::strtoul(color.substr(1 + c * 2, 2).c_str(), nullptr, 16));
+        for (int c = 0; c < channels; ++c) {
+            float fill = rgba[channels == 1 ? 0 : c] / 255.f;
+            for (int y = 0; y < p.height; ++y) {
+                for (int x = 0; x < p.width; ++x) {
+                    output.index(x, y, c, 0) = x >= p.pad_x && x < p.pad_x + p.resize_width && y >= p.pad_y && y < p.pad_y + p.resize_height
+                                                   ? resized.index(x - p.pad_x, y - p.pad_y, c, 0)
+                                                   : fill;
+                }
+            }
+        }
+        LOG_INFO("preprocess %s: %dx%d crop=(%d,%d,%d,%d) resize=%dx%d pad=(%d,%d) output=%dx%d filter=%s(%d) antialias=%s",
+                 label.c_str(), p.source_width, p.source_height, p.x, p.y, p.crop_width, p.crop_height,
+                 p.resize_width, p.resize_height, p.pad_x, p.pad_y, p.width, p.height, filter.c_str(), static_cast<int>(mode), BOOL_STR(antialias));
+        return output;
+    }
+
+    Tensor<float> ImagePreprocessor::apply_geometry(const Tensor<float>& image, ImageTarget target, int index, int width, int height, ImageResizeMode default_mode, ops::InterpolateMode default_filter, ImageTransform* plan_out) const {
+        if (!valid_ || image.empty())
+            return {};
+        const std::string label = std::string(enum_name(target, image_targets)) + "[" + std::to_string(index) + "]";
+        auto options            = resolve_options(target, index);
+        if (image.dim() != 4 || image.shape()[3] != 1 || image.shape()[2] < 1 || image.shape()[2] > 4) {
+            fail(label + ": expected one image with 1 to 4 channels");
+            return {};
+        }
+        ImageTransform p;
+        p.source_width = p.crop_width = static_cast<int>(image.shape()[0]);
+        p.source_height = p.crop_height = static_cast<int>(image.shape()[1]);
+        int target_width                = width > 0 ? width : p.source_width;
+        int target_height               = height > 0 ? height : p.source_height;
+        if (options.count("width")) {
+            parse_strict_int(options.at("width"), target_width);
+            parse_strict_int(options.at("height"), target_height);
+        }
+        ImageResizeMode mode = resolve_mode(options, default_mode);
+        std::string anchor   = options.count("anchor") ? options.at("anchor") : "center";
+        p.width = p.resize_width = target_width;
+        p.height = p.resize_height = target_height;
+        if (mode == ImageResizeMode::None) {
+            if (options.count("width") && (target_width != p.source_width || target_height != p.source_height)) {
+                fail(label + ": mode=none conflicts with requested dimensions");
+                return {};
+            }
+            p.width = p.resize_width = p.source_width;
+            p.height = p.resize_height = p.source_height;
+        } else if (mode == ImageResizeMode::Crop || mode == ImageResizeMode::CropResize) {
+            if (mode == ImageResizeMode::Crop) {
+                p.crop_width  = target_width;
+                p.crop_height = target_height;
+            } else if (int64_t(p.source_width) * target_height > int64_t(p.source_height) * target_width) {
+                p.crop_width = std::max(1, static_cast<int>(int64_t(p.source_height) * target_width / target_height));
+            } else {
+                p.crop_height = std::max(1, static_cast<int>(int64_t(p.source_width) * target_height / target_width));
+            }
+            if (p.crop_width > p.source_width || p.crop_height > p.source_height) {
+                fail(label + ": crop exceeds source dimensions");
+                return {};
+            }
+            p.x = anchor_offset(p.source_width - p.crop_width, anchor, true);
+            p.y = anchor_offset(p.source_height - p.crop_height, anchor, false);
+        } else if (mode == ImageResizeMode::FitPad) {
+            double scale    = std::min(double(target_width) / p.source_width, double(target_height) / p.source_height);
+            p.resize_width  = std::max(1, std::min(target_width, static_cast<int>(std::round(p.source_width * scale))));
+            p.resize_height = std::max(1, std::min(target_height, static_cast<int>(std::round(p.source_height * scale))));
+            p.pad_x         = anchor_offset(target_width - p.resize_width, anchor, true);
+            p.pad_y         = anchor_offset(target_height - p.resize_height, anchor, false);
+        }
+        if (p.width <= 0 || p.height <= 0) {
+            fail(label + ": invalid output dimensions");
+            return {};
+        }
+        uint64_t max_pixels = std::min<uint64_t>(INT64_MAX, SIZE_MAX / sizeof(float)) / static_cast<uint64_t>(image.shape()[2]);
+        if (uint64_t(p.width) * p.height > max_pixels || uint64_t(p.resize_width) * p.resize_height > max_pixels) {
+            fail(label + ": image allocation size overflows");
+            return {};
+        }
+        if (plan_out != nullptr)
+            *plan_out = p;
+        return apply_transform(image, options, p, label, default_filter);
+    }
+
+    Tensor<float> ImagePreprocessor::preprocess_input(sd_image_t image, ImageTarget target, int index, int width, int height) {
+        if (image.data == nullptr || image.width == 0 || image.height == 0 || image.width > INT_MAX || image.height > INT_MAX || image.channel < 1 || image.channel > 4) {
+            fail(std::string(enum_name(target, image_targets)) + ": invalid input image");
+            return {};
+        }
+        auto tensor = sd_image_to_tensor(image);
+        if (target == ImageTarget::Mask && has_init_transform_) {
+            auto options = resolve_options(target, index);
+            if (image.width != init_transform_.source_width || image.height != init_transform_.source_height) {
+                fail("mask and init source dimensions must match");
+                return {};
+            }
+            bool geometry_override = options.count("width") || options.count("anchor") ||
+                                     (options.count("mode") && options.at("mode") != "auto");
+            if (geometry_override) {
+                ImageTransform p;
+                auto init_options            = resolve_options(ImageTarget::Init, 0);
+                ImageResizeMode default_mode = resolve_mode(init_options, ImageResizeMode::CropResize);
+                auto result                  = apply_geometry(tensor, target, index, init_transform_.width, init_transform_.height, default_mode, ops::InterpolateMode::NearestExact, &p);
+                if (result.empty())
+                    return {};
+                const auto& q = init_transform_;
+                if (p.x != q.x || p.y != q.y || p.crop_width != q.crop_width || p.crop_height != q.crop_height ||
+                    p.resize_width != q.resize_width || p.resize_height != q.resize_height || p.pad_x != q.pad_x || p.pad_y != q.pad_y || p.width != q.width || p.height != q.height) {
+                    fail("mask geometry conflicts with init; configure geometry on init and filter on mask");
+                    return {};
+                }
+                return result;
+            }
+            return apply_transform(tensor, options, init_transform_, "mask[0]", ops::InterpolateMode::NearestExact);
+        }
+        auto result = apply_geometry(tensor, target, index, width, height, width > 0 ? ImageResizeMode::CropResize : ImageResizeMode::None,
+                                     target == ImageTarget::Mask ? ops::InterpolateMode::NearestExact : ops::InterpolateMode::Nearest,
+                                     target == ImageTarget::Init ? &init_transform_ : nullptr);
+        if (target == ImageTarget::Init)
+            has_init_transform_ = !result.empty();
+        return result;
+    }
+
+    ImagePreprocessor::~ImagePreprocessor() {
+        for (const auto& image : owned_images_)
+            std::free(image.data);
+    }
+
+    bool ImagePreprocessor::prepare_image(sd_image_t& image, ImageTarget target, int index, int width, int height) {
+        if (image.data == nullptr)
+            return true;
+        auto options = resolve_options(target, index);
+        bool canny   = options.count("canny") && options.at("canny") == "true";
+        auto tensor  = preprocess_input(image, target, index, width, height);
+        if (tensor.empty())
+            return false;
+        auto output = tensor_to_sd_image(tensor);
+        if (output.data == nullptr)
+            return fail("could not allocate input preprocessing buffer");
+        owned_images_.push_back(output);
+        if (canny && !preprocess_canny(output, 0.08f, 0.08f, 0.8f, 1.f, false))
+            return fail("Canny preprocessing failed");
+        image = output;
+        return true;
+    }
+
+    bool ImagePreprocessor::prepare_array(sd_image_t*& images, int count, ImageTarget target, std::vector<sd_image_t>& storage, int width, int height) {
+        if (count < 0 || (count > 0 && images == nullptr))
+            return fail(std::string("invalid image array: ") + enum_name(target, image_targets));
+        if (count == 0)
+            return true;
+        storage.assign(images, images + count);
+        for (int i = 0; i < count; ++i) {
+            if (storage[i].data == nullptr)
+                return fail(std::string("empty image in array: ") + enum_name(target, image_targets));
+            if (!prepare_image(storage[i], target, i, width, height))
+                return false;
+        }
+        images = storage.data();
+        return true;
+    }
+
+    bool ImagePreprocessor::prepare_inputs(sd_img_gen_params_t& params, int width, int height) {
+        if (prepared_)
+            return fail("inputs have already been prepared");
+        prepared_ = true;
+        if (!valid_ || !validate_inputs(params))
+            return false;
+        if (!prepare_image(params.init_image, ImageTarget::Init, 0, width, height) ||
+            !prepare_image(params.mask_image, ImageTarget::Mask, 0, width, height) ||
+            !prepare_image(params.control_image, ImageTarget::Control, 0, width, height) ||
+            !prepare_image(params.ip_adapter_image, ImageTarget::IPAdapter, 0, -1, -1) ||
+            !prepare_array(params.ref_images, params.ref_images_count, ImageTarget::Ref, ref_images_) ||
+            !prepare_array(params.pm_params.id_images, params.pm_params.id_images_count, ImageTarget::ID, id_images_))
+            return false;
+        params.image_preprocess = {};
+        return true;
+    }
+
+    bool ImagePreprocessor::prepare_inputs(sd_vid_gen_params_t& params, int width, int height) {
+        if (prepared_)
+            return fail("inputs have already been prepared");
+        prepared_ = true;
+        if (!valid_ || !validate_inputs(params))
+            return false;
+        if (!prepare_image(params.init_image, ImageTarget::Init, 0, width, height) ||
+            !prepare_image(params.end_image, ImageTarget::End, 0, width, height) ||
+            !prepare_array(params.ref_images, params.ref_images_count, ImageTarget::Ref, ref_images_) ||
+            !prepare_array(params.control_frames, params.control_frames_size, ImageTarget::ControlFrame, control_frames_, width, height))
+            return false;
+        params.image_preprocess = {};
+        return true;
+    }
+
+}  // namespace sd
diff --git a/src/runtime/image_preprocess.h b/src/runtime/image_preprocess.h
new file mode 100644
index 0000000000..b75c4e0e50
--- /dev/null
+++ b/src/runtime/image_preprocess.h
@@ -0,0 +1,88 @@
+#ifndef __SD_RUNTIME_IMAGE_PREPROCESS_H__
+#define __SD_RUNTIME_IMAGE_PREPROCESS_H__
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "core/tensor.hpp"
+#include "stable-diffusion.h"
+
+namespace sd {
+
+    enum class ImageTarget {
+        Init,
+        End,
+        Mask,
+        Control,
+        Ref,
+        IPAdapter,
+        ID,
+        ControlFrame,
+    };
+
+    enum class ImageResizeMode {
+        Auto,
+        None,
+        Stretch,
+        Crop,
+        CropResize,
+        FitPad,
+    };
+
+    struct ImageTransform {
+        int source_width  = 0;
+        int source_height = 0;
+        int x             = 0;
+        int y             = 0;
+        int crop_width    = 0;
+        int crop_height   = 0;
+        int resize_width  = 0;
+        int resize_height = 0;
+        int width         = 0;
+        int height        = 0;
+        int pad_x         = 0;
+        int pad_y         = 0;
+    };
+
+    struct ImagePreprocessRule {
+        ImageTarget target = ImageTarget::Init;
+        int index          = -1;
+        std::map<std::string, std::string> options;
+    };
+
+    class ImagePreprocessor {
+        std::vector<ImagePreprocessRule> rules_;
+        mutable bool valid_ = true;
+        ImageTransform init_transform_;
+        bool has_init_transform_ = false;
+        bool prepared_           = false;
+        std::vector<sd_image_t> owned_images_;
+        std::vector<sd_image_t> ref_images_;
+        std::vector<sd_image_t> id_images_;
+        std::vector<sd_image_t> control_frames_;
+
+        bool fail(const std::string& message) const;
+        std::map<std::string, std::string> resolve_options(ImageTarget target, int index) const;
+        Tensor<float> apply_transform(const Tensor<float>& image, const std::map<std::string, std::string>& options, ImageTransform plan, const std::string& label, ops::InterpolateMode default_filter) const;
+
+        bool prepare_image(sd_image_t& image, ImageTarget target, int index, int width, int height);
+        bool prepare_array(sd_image_t*& images, int count, ImageTarget target, std::vector<sd_image_t>& storage, int width = -1, int height = -1);
+
+    public:
+        explicit ImagePreprocessor(const char* rules = nullptr);
+        ~ImagePreprocessor();
+        ImagePreprocessor(const ImagePreprocessor&)            = delete;
+        ImagePreprocessor& operator=(const ImagePreprocessor&) = delete;
+        bool prepare_inputs(sd_img_gen_params_t& params, int width, int height);
+        bool prepare_inputs(sd_vid_gen_params_t& params, int width, int height);
+        bool is_valid() const { return valid_; }
+        bool validate_inputs(const sd_img_gen_params_t& params) const;
+        bool validate_inputs(const sd_vid_gen_params_t& params) const;
+        Tensor<float> apply_geometry(const Tensor<float>& image, ImageTarget target, int index, int width, int height, ImageResizeMode default_mode = ImageResizeMode::Stretch, ops::InterpolateMode default_filter = ops::InterpolateMode::Nearest, ImageTransform* plan_out = nullptr) const;
+        Tensor<float> preprocess_input(sd_image_t image, ImageTarget target, int index = 0, int width = -1, int height = -1);
+    };
+
+}  // namespace sd
+
+#endif  // __SD_RUNTIME_IMAGE_PREPROCESS_H__
diff --git a/src/runtime/imatrix.cpp b/src/runtime/imatrix.cpp
index a6f3bca950..185832cb2c 100644
--- a/src/runtime/imatrix.cpp
+++ b/src/runtime/imatrix.cpp
@@ -10,6 +10,7 @@
 #include <cmath>
 #include <cstdlib>
 #include <cstring>
+#include <filesystem>
 
 static IMatrixCollector imatrix_collector;
 
@@ -67,15 +68,56 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor* t, bool ask, void* us
     }
     std::lock_guard<std::mutex> lock(mutex_);
 
-    // copy the data from the GPU memory if needed
+    ++layout_.calls;
+    // Rows are read through nb[1..3], so any layout whose rows are packed elements works; a strided
+    // nb[0] or an element type with no float conversion here is refused, counted, and reported at save time.
+    const bool type_ok = src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16;
+    if (!type_ok || src1->nb[0] != ggml_type_size(src1->type)) {
+        ++layout_.rejected;
+        LOG_ERROR("imatrix: cannot collect '%s': activation type %s with element stride %zu is not supported; this tensor gets no data from this call\n",
+                  wname.c_str(), ggml_type_name(src1->type), (size_t)src1->nb[0]);
+        return true;
+    }
+    if (src1->type == GGML_TYPE_F16) {
+        ++layout_.f16;
+    } else if (src1->type == GGML_TYPE_BF16) {
+        ++layout_.bf16;
+    }
+    if (!ggml_is_contiguous(src1)) {
+        ++layout_.non_contiguous;
+    }
+    if (src1->ne[2] * src1->ne[3] > 1) {
+        ++layout_.batched;
+    }
+    if (src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && src1->ne[2] * src1->ne[3] == 1) {
+        ++layout_.contiguous_f32_2d;
+    }
+
+    // copy the data from the GPU memory if needed; ggml_nbytes spans the whole (possibly strided) view
     const bool is_host = src1->buffer == NULL || ggml_backend_buffer_is_host(src1->buffer);
 
     if (!is_host) {
-        src1_data_.resize(ggml_nelements(src1));
-        ggml_backend_tensor_get(src1, src1_data_.data(), 0, ggml_nbytes(src1));
+        src1_bytes_.resize(ggml_nbytes(src1));
+        ggml_backend_tensor_get(src1, src1_bytes_.data(), 0, ggml_nbytes(src1));
     }
 
-    const float* data = is_host ? (const float*)src1->data : src1_data_.data();
+    const char* data  = is_host ? (const char*)src1->data : src1_bytes_.data();
+    const int64_t ne0 = src1->ne[0];
+    row_f32_.resize(ne0);
+
+    // One activation row as f32, addressed through the byte strides.
+    auto row_at = [&](int64_t i1, int64_t i2, int64_t i3) -> const float* {
+        const char* p = data + i1 * src1->nb[1] + i2 * src1->nb[2] + i3 * src1->nb[3];
+        if (src1->type == GGML_TYPE_F32) {
+            return (const float*)p;
+        }
+        if (src1->type == GGML_TYPE_F16) {
+            ggml_fp16_to_fp32_row((const ggml_fp16_t*)p, row_f32_.data(), ne0);
+        } else {
+            ggml_bf16_to_fp32_row((const ggml_bf16_t*)p, row_f32_.data(), ne0);
+        }
+        return row_f32_.data();
+    };
 
     // this has been adapted to the new format of storing merged experts in a single 3d tensor
     // ref: https://github.com/ggml-org/llama.cpp/pull/6387
@@ -121,7 +163,7 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor* t, bool ask, void* us
 
                     const int64_t i11 = idx % src1->ne[1];
                     const int64_t i12 = row;
-                    const float* x    = (const float*)((const char*)data + i11 * src1->nb[1] + i12 * src1->nb[2]);
+                    const float* x    = row_at(i11, i12, 0);
 
                     for (int j = 0; j < (int)src1->ne[0]; ++j) {
                         e.values[e_start + j] += x[j] * x[j];
@@ -145,18 +187,24 @@ bool IMatrixCollector::collect_imatrix(struct ggml_tensor* t, bool ask, void* us
         }
 
         ++e.ncall;
-        for (int row = 0; row < (int)src1->ne[1]; ++row) {
-            const float* x = data + row * src1->ne[0];
-            for (int j = 0; j < (int)src1->ne[0]; ++j) {
-                if (std::isfinite(x[j])) {
-                    e.values[j] += x[j] * x[j];
-                    e.counts[j]++;
-                    if (!std::isfinite(e.values[j])) {
-                        LOG_WARN("%f detected in %s\n", e.values[j], wname.c_str());
-                        exit(1);
+        // Every row of every plane: src0 broadcasts over src1's ne[2]/ne[3], and each plane's rows hit the
+        // same ne[0] input columns of the weight, so all of them accumulate into the one per-column vector.
+        for (int64_t i3 = 0; i3 < src1->ne[3]; ++i3) {
+            for (int64_t i2 = 0; i2 < src1->ne[2]; ++i2) {
+                for (int64_t row = 0; row < src1->ne[1]; ++row) {
+                    const float* x = row_at(row, i2, i3);
+                    for (int64_t j = 0; j < ne0; ++j) {
+                        if (std::isfinite(x[j])) {
+                            e.values[j] += x[j] * x[j];
+                            e.counts[j]++;
+                            if (!std::isfinite(e.values[j])) {
+                                LOG_WARN("%f detected in %s\n", e.values[j], wname.c_str());
+                                exit(1);
+                            }
+                        } else {
+                            // Likely something from an attention mask?
+                        }
                     }
-                } else {
-                    // Likely something from an attention mask?
                 }
             }
         }
@@ -223,11 +271,17 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const {
         to_store.push_back(kv.first);
     }
 
+    LOG_INFO("imatrix: %lld matmul calls collected (contiguous f32 2-D %lld, non-contiguous %lld, batched ne[2]/ne[3] %lld, f16 %lld, bf16 %lld, rejected %lld)\n",
+             layout_.calls, layout_.contiguous_f32_2d, layout_.non_contiguous, layout_.batched, layout_.f16, layout_.bf16, layout_.rejected);
+    if (layout_.rejected > 0) {
+        LOG_ERROR("imatrix: %lld matmul calls were not collected (unsupported activation layout), see the errors above\n", layout_.rejected);
+    }
+
     if (to_store.size() < stats_.size()) {
         LOG_WARN("storing only %zu out of %zu entries\n", to_store.size(), stats_.size());
     }
 
-    std::ofstream out(fname, std::ios::binary);
+    std::ofstream out(std::filesystem::u8path(fname), std::ios::binary);
     out.write((const char*)&n_entries, sizeof(n_entries));
     for (const auto& name : to_store) {
         const auto& stat = stats_.at(name);
@@ -251,7 +305,7 @@ void IMatrixCollector::save_imatrix(std::string fname, int ncall) const {
 }
 
 bool IMatrixCollector::load_imatrix(const char* fname) {
-    std::ifstream in(fname, std::ios::binary);
+    std::ifstream in(std::filesystem::u8path(fname), std::ios::binary);
     if (!in) {
         LOG_ERROR("failed to open %s\n", fname);
         return false;
diff --git a/src/runtime/imatrix.h b/src/runtime/imatrix.h
index 341c18c8bc..18dcb5b53b 100644
--- a/src/runtime/imatrix.h
+++ b/src/runtime/imatrix.h
@@ -17,12 +17,25 @@ struct IMatrixStats {
     int ncall = 0;
 };
 
+// How many collected matmuls saw each src1 layout; reported when the imatrix is saved.
+struct IMatrixLayoutCounts {
+    long long calls             = 0;
+    long long contiguous_f32_2d = 0;
+    long long non_contiguous    = 0;
+    long long batched           = 0;
+    long long f16               = 0;
+    long long bf16              = 0;
+    long long rejected          = 0;
+};
+
 class IMatrixCollector {
 private:
     std::unordered_map<std::string, IMatrixStats> stats_ = {};
     std::mutex mutex_;
     int last_call_ = 0;
-    std::vector<float> src1_data_;
+    std::vector<char> src1_bytes_;
+    std::vector<float> row_f32_;
+    IMatrixLayoutCounts layout_;
     std::vector<char> ids_;  // the expert ids from ggml_mul_mat_id
 
 public:
@@ -30,6 +43,7 @@ class IMatrixCollector {
     bool collect_imatrix(struct ggml_tensor* t, bool ask, void* user_data);
     void save_imatrix(std::string fname, int ncall = -1) const;
     bool load_imatrix(const char* fname);
+    const IMatrixLayoutCounts& layout_counts() const { return layout_; }
     std::vector<float> get_values(const std::string& key) const {
         auto it = stats_.find(key);
         if (it != stats_.end()) {
diff --git a/src/runtime/latent-preview.h b/src/runtime/latent-preview.h
index c19a0eebcf..5baf6f53cb 100644
--- a/src/runtime/latent-preview.h
+++ b/src/runtime/latent-preview.h
@@ -4,6 +4,86 @@
 #include "core/tensor.hpp"
 #include "ggml.h"
 
+// RGB is projected to [-1, 1]; alpha is projected directly to [0, 1].
+const float qwen21_latent_rgb_proj[64][3] = {
+    {0.00860495522f, 0.01219501462f, -0.00321337196f},
+    {0.01889233090f, 0.01246581216f, 0.01074959482f},
+    {0.1255941446f, 0.1176879344f, -0.0332212352f},
+    {0.0418238528f, 0.1043427754f, 0.0121666316f},
+    {0.02025338f, 0.01453670296f, -0.000224336044f},
+    {-0.01896720702f, -0.0206099030f, -0.0322728584f},
+    {0.00438984796f, -0.01374969766f, 0.02849196f},
+    {-0.0374495856f, -0.0286777126f, -0.0693192810f},
+    {0.01511914734f, 0.0242979386f, 0.0553878870f},
+    {-0.1138629518f, -0.020391466f, 0.001550520522f},
+    {-0.0233650696f, -0.0417292018f, -0.0362361182f},
+    {-0.0351603342f, -0.0243595924f, -0.00216261038f},
+    {0.01093355288f, -0.0373466924f, 0.00241315350f},
+    {0.01778704744f, -0.00401984678f, -0.0343259192f},
+    {0.0486059334f, 0.0253144f, 0.0672564966f},
+    {0.0309463558f, 0.0277963166f, 0.0520869622f},
+    {0.0374485008f, 0.0551753676f, 0.0225853902f},
+    {-0.0090809962f, -0.004756176f, 0.00636443612f},
+    {-0.0270455652f, -0.0384966954f, -0.00905908082f},
+    {-0.00553493756f, 0.01484553684f, -0.0211502468f},
+    {0.01319502562f, 0.00948005666f, 0.0483789212f},
+    {-0.00931847104f, -0.00276452734f, -0.01011985302f},
+    {0.0180478258f, 0.01614954356f, -0.0209424690f},
+    {-0.0214530434f, -0.00272961176f, 0.0217887476f},
+    {-0.0636772304f, -0.0208893548f, 0.0479167742f},
+    {-0.0250321236f, -0.0286715676f, 0.0530110146f},
+    {-0.01853078078f, 0.01647272818f, -0.00207747588f},
+    {0.0023101082f, 0.01228800748f, 0.01303505006f},
+    {-0.01243671408f, -0.0258638728f, -0.0379116264f},
+    {0.00598934710f, 0.00642563550f, -0.01234514304f},
+    {-0.0296733996f, -0.0234698050f, 0.00060018212f},
+    {-0.0322019498f, -0.0529200462f, -0.00344987414f},
+    {-0.00205026458f, -0.00846599446f, 0.00455971038f},
+    {-0.01082227064f, 0.0315661948f, -0.0677753362f},
+    {0.0645553474f, 0.1109666998f, 0.0674744864f},
+    {0.01036801108f, -0.00484841210f, -0.001529168474f},
+    {0.01264353566f, 0.01548126338f, -0.00966374324f},
+    {-0.0223892408f, -0.00871751526f, -0.000306421670f},
+    {0.0271322742f, 0.03496524f, -0.0089692858f},
+    {0.0512178672f, 0.0173080034f, 0.00804227746f},
+    {0.01210987192f, 0.00758025926f, -0.00281712586f},
+    {0.1897278390f, 0.1210261828f, 0.062603892f},
+    {0.0208058822f, 0.00547548182f, 0.01262955638f},
+    {0.00813332858f, 0.01015930914f, 0.01301771290f},
+    {-0.000927236014f, -0.00152540594f, -0.00599213302f},
+    {0.01663314616f, -0.00582789626f, 0.0163958132f},
+    {-0.0252546342f, -0.0604193732f, -0.1606919922f},
+    {-0.091722686f, -0.0409201224f, -0.0959576198f},
+    {0.0282963112f, -0.01387223872f, -0.01648814464f},
+    {0.0552316818f, 0.0967547788f, 0.0413586632f},
+    {0.00922849292f, 0.00451467542f, -0.0529172378f},
+    {0.0558600768f, 0.0122988308f, -0.01445942422f},
+    {0.000210660902f, -0.01295958782f, -0.01804761764f},
+    {0.0358136250f, -0.0472505970f, -0.1156405142f},
+    {-0.0506390696f, -0.0471914842f, 0.0349791468f},
+    {-0.0480143168f, 0.00628389868f, -0.0545163826f},
+    {0.0315499582f, 0.0564846606f, -0.0430850488f},
+    {-0.0362330316f, -0.01267788554f, 0.0061024772f},
+    {0.0038627542f, 0.00911055916f, -0.00758526008f},
+    {-0.0447103298f, -0.00835411408f, 0.01545872328f},
+    {-0.015006738f, 0.00270612302f, -0.00784361356f},
+    {-0.0221755048f, -0.0513344748f, -0.0475317424f},
+    {-0.01036656294f, -0.00422146068f, -0.0213499052f},
+    {0.01788952706f, 0.01191944190f, 0.0397205238f},
+};
+float qwen21_latent_rgb_bias[3] = {-0.043293118f, -0.02695978f, -0.11986706f};
+
+const float qwen21_latent_alpha_proj[64] = {
+    -0.0416241114f, -0.00678954612f, -0.0169095515f, -0.0230551401f, 0.0100882595f, 0.00655586802f, 0.0401166874f, -0.0055510216f,
+    0.0224234441f, -0.0389640963f, -0.0114492163f, -0.00721128977f, -0.0029064082f, 0.0150300547f, -0.00321615308f, -0.0498856338f,
+    -0.0215251401f, 0.0240220482f, 0.0117338008f, -0.0460420624f, 0.0387872889f, 0.0131517207f, 0.0147100836f, 0.0266985286f,
+    0.0153097324f, -0.0418119757f, 0.0421013917f, 0.0401724499f, 0.00972515915f, 0.011718495f, 0.0117622291f, 0.0136505134f,
+    -0.0350017363f, -0.0100692606f, -0.0131582529f, -0.00660639315f, 0.00253611396f, -0.0195736368f, -0.04240184f, 0.0321299262f,
+    0.0106089414f, -0.0179845306f, -0.00806212708f, 0.0135889057f, -0.0157393098f, -0.0267791344f, 0.0109068534f, 0.0283931966f,
+    -0.0435370078f, 0.00187883536f, -0.0108995378f, -0.0450757676f, -0.0699481501f, 0.0123562106f, -0.0222592249f, 0.0216155907f,
+    0.0563755424f, -0.0073379912f, 0.0160012921f, 0.0411637742f, 0.0189607258f, -0.024025029f, -0.0161487905f, -0.016913203f};
+const float qwen21_latent_alpha_bias = 0.871268134f;
+
 const float minimax_latent_rgb_proj[24][3] = {
     {0.19819857f, 0.11584999f, 0.07929777f},
     {-0.16047224f, -0.10601170f, -0.15996324f},
@@ -324,7 +404,7 @@ const float sd_latent_rgb_proj[4][3] = {
     {-0.178022f, -0.200862f, -0.678514f}};
 float sd_latent_rgb_bias[3] = {-0.017478f, -0.055834f, -0.105825f};
 
-void preview_latent_video(uint8_t* buffer, ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) {
+void preview_latent_video(uint8_t* buffer, ggml_tensor* latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size, const float* latent_alpha_proj = nullptr, float latent_alpha_bias = 1.f) {
     size_t buffer_head = 0;
 
     uint32_t latent_width  = static_cast<uint32_t>(latents->ne[0]);
@@ -338,7 +418,8 @@ void preview_latent_video(uint8_t* buffer, ggml_tensor* latents, const float (*l
     uint32_t rgb_width  = latent_width * patch_size;
     uint32_t rgb_height = latent_height * patch_size;
 
-    uint32_t unpatched_dim = dim / (patch_size * patch_size);
+    uint32_t unpatched_dim         = dim / (patch_size * patch_size);
+    const uint32_t output_channels = latent_alpha_proj != nullptr ? 4 : 3;
 
     for (uint32_t k = 0; k < frames; k++) {
         for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) {
@@ -356,13 +437,16 @@ void preview_latent_video(uint8_t* buffer, ggml_tensor* latents, const float (*l
                 // should be incremented by 1 for each pixel
                 size_t pixel_id = k * rgb_width * rgb_height + rgb_y * rgb_width + rgb_x;
 
-                float r = 0, g = 0, b = 0;
+                float r = 0, g = 0, b = 0, a = 0;
                 if (latent_rgb_proj != nullptr) {
                     for (uint32_t d = 0; d < unpatched_dim; d++) {
                         float value = *(float*)((char*)latents->data + latent_id + (d * patch_size * patch_size + channel_offset) * latents->nb[ggml_n_dims(latents) - 1]);
                         r += value * latent_rgb_proj[d][0];
                         g += value * latent_rgb_proj[d][1];
                         b += value * latent_rgb_proj[d][2];
+                        if (latent_alpha_proj != nullptr) {
+                            a += value * latent_alpha_proj[d];
+                        }
                     }
                 } else {
                     // interpret first 3 channels as RGB
@@ -386,9 +470,13 @@ void preview_latent_video(uint8_t* buffer, ggml_tensor* latents, const float (*l
                 g = g >= 0 ? g <= 1 ? g : 1 : 0;
                 b = b >= 0 ? b <= 1 ? b : 1 : 0;
 
-                buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255);
-                buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255);
-                buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255);
+                buffer[pixel_id * output_channels + 0] = (uint8_t)(r * 255);
+                buffer[pixel_id * output_channels + 1] = (uint8_t)(g * 255);
+                buffer[pixel_id * output_channels + 2] = (uint8_t)(b * 255);
+                if (latent_alpha_proj != nullptr) {
+                    a                                      = std::min(1.0f, std::max(0.0f, a + latent_alpha_bias));
+                    buffer[pixel_id * output_channels + 3] = (uint8_t)(a * 255);
+                }
             }
         }
     }
@@ -398,16 +486,17 @@ static inline bool preview_latent_tensor_is_video(const sd::Tensor<float>& laten
     return latents.dim() == 5;
 }
 
-void preview_latent_video(uint8_t* buffer, const sd::Tensor<float>& latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size) {
+void preview_latent_video(uint8_t* buffer, const sd::Tensor<float>& latents, const float (*latent_rgb_proj)[3], const float latent_rgb_bias[3], int patch_size, const float* latent_alpha_proj = nullptr, float latent_alpha_bias = 1.f) {
     uint32_t latent_width  = static_cast<uint32_t>(latents.shape()[0]);
     uint32_t latent_height = static_cast<uint32_t>(latents.shape()[1]);
     bool is_video          = preview_latent_tensor_is_video(latents);
     uint32_t frames        = is_video ? static_cast<uint32_t>(latents.shape()[2]) : 1;
     uint32_t dim           = is_video ? static_cast<uint32_t>(latents.shape()[3]) : static_cast<uint32_t>(latents.shape()[2]);
 
-    uint32_t rgb_width     = latent_width * patch_size;
-    uint32_t rgb_height    = latent_height * patch_size;
-    uint32_t unpatched_dim = dim / (patch_size * patch_size);
+    uint32_t rgb_width             = latent_width * patch_size;
+    uint32_t rgb_height            = latent_height * patch_size;
+    uint32_t unpatched_dim         = dim / (patch_size * patch_size);
+    const uint32_t output_channels = latent_alpha_proj != nullptr ? 4 : 3;
 
     for (uint32_t k = 0; k < frames; k++) {
         for (uint32_t rgb_x = 0; rgb_x < rgb_width; rgb_x++) {
@@ -427,7 +516,7 @@ void preview_latent_video(uint8_t* buffer, const sd::Tensor<float>& latents, con
                                : latents.values()[latent_x + latent_width * (latent_y + latent_height * latent_channel)];
                 };
 
-                float r = 0.f, g = 0.f, b = 0.f;
+                float r = 0.f, g = 0.f, b = 0.f, a = 0.f;
                 if (latent_rgb_proj != nullptr) {
                     for (uint32_t d = 0; d < unpatched_dim; d++) {
                         uint32_t latent_channel = d * patch_size * patch_size + channel_offset;
@@ -435,6 +524,9 @@ void preview_latent_video(uint8_t* buffer, const sd::Tensor<float>& latents, con
                         r += value * latent_rgb_proj[d][0];
                         g += value * latent_rgb_proj[d][1];
                         b += value * latent_rgb_proj[d][2];
+                        if (latent_alpha_proj != nullptr) {
+                            a += value * latent_alpha_proj[d];
+                        }
                     }
                 } else {
                     r = latent_value(0);
@@ -450,9 +542,13 @@ void preview_latent_video(uint8_t* buffer, const sd::Tensor<float>& latents, con
                 g = std::min(1.0f, std::max(0.0f, g * .5f + .5f));
                 b = std::min(1.0f, std::max(0.0f, b * .5f + .5f));
 
-                buffer[pixel_id * 3 + 0] = (uint8_t)(r * 255);
-                buffer[pixel_id * 3 + 1] = (uint8_t)(g * 255);
-                buffer[pixel_id * 3 + 2] = (uint8_t)(b * 255);
+                buffer[pixel_id * output_channels + 0] = (uint8_t)(r * 255);
+                buffer[pixel_id * output_channels + 1] = (uint8_t)(g * 255);
+                buffer[pixel_id * output_channels + 2] = (uint8_t)(b * 255);
+                if (latent_alpha_proj != nullptr) {
+                    a                                      = std::min(1.0f, std::max(0.0f, a + latent_alpha_bias));
+                    buffer[pixel_id * output_channels + 3] = (uint8_t)(a * 255);
+                }
             }
         }
     }
diff --git a/src/runtime/preprocessing.hpp b/src/runtime/preprocessing.hpp
index 1990d15f46..ab4ca29b64 100644
--- a/src/runtime/preprocessing.hpp
+++ b/src/runtime/preprocessing.hpp
@@ -165,16 +165,18 @@ static inline sd::Tensor<float> convolve_tensor(const sd::Tensor<float>& input,
     return output;
 }
 
-static inline sd::Tensor<float> grayscale_tensor(const sd::Tensor<float>& rgb_img) {
-    GGML_ASSERT(rgb_img.dim() == 4);
-    GGML_ASSERT(rgb_img.shape()[2] >= 3);
-    sd::Tensor<float> grayscale({rgb_img.shape()[0], rgb_img.shape()[1], 1, rgb_img.shape()[3]});
-    for (int64_t iy = 0; iy < rgb_img.shape()[1]; ++iy) {
-        for (int64_t ix = 0; ix < rgb_img.shape()[0]; ++ix) {
-            float r    = preprocessing_get_4d(rgb_img, ix, iy, 0, 0);
-            float g    = preprocessing_get_4d(rgb_img, ix, iy, 1, 0);
-            float b    = preprocessing_get_4d(rgb_img, ix, iy, 2, 0);
-            float gray = 0.2989f * r + 0.5870f * g + 0.1140f * b;
+static inline sd::Tensor<float> grayscale_tensor(const sd::Tensor<float>& image) {
+    GGML_ASSERT(image.dim() == 4);
+    GGML_ASSERT(image.shape()[2] >= 1);
+    sd::Tensor<float> grayscale({image.shape()[0], image.shape()[1], 1, image.shape()[3]});
+    for (int64_t iy = 0; iy < image.shape()[1]; ++iy) {
+        for (int64_t ix = 0; ix < image.shape()[0]; ++ix) {
+            float gray = preprocessing_get_4d(image, ix, iy, 0, 0);
+            if (image.shape()[2] >= 3) {
+                float g = preprocessing_get_4d(image, ix, iy, 1, 0);
+                float b = preprocessing_get_4d(image, ix, iy, 2, 0);
+                gray    = 0.2989f * gray + 0.5870f * g + 0.1140f * b;
+            }
             preprocessing_set_4d(grayscale, gray, ix, iy, 0, 0);
         }
     }
@@ -317,11 +319,12 @@ bool preprocess_canny(sd_image_t img, float high_threshold, float low_threshold,
     image_gray              = non_max_supression(G, theta);
     threshold_hystersis(&image_gray, high_threshold, low_threshold, weak, strong);
 
+    const uint32_t color_channels = img.channel == 2 || img.channel == 4 ? img.channel - 1 : img.channel;
     for (uint32_t iy = 0; iy < img.height; ++iy) {
         for (uint32_t ix = 0; ix < img.width; ++ix) {
             float gray = preprocessing_get_4d(image_gray, ix, iy, 0, 0);
             gray       = inverse ? 1.0f - gray : gray;
-            for (uint32_t c = 0; c < img.channel; ++c) {
+            for (uint32_t c = 0; c < color_channels; ++c) {
                 preprocessing_set_4d(image, gray, ix, iy, c, 0);
             }
         }
diff --git a/src/runtime/sample-cache.cpp b/src/runtime/sample-cache.cpp
index bfa3d4487f..0b92c73358 100644
--- a/src/runtime/sample-cache.cpp
+++ b/src/runtime/sample-cache.cpp
@@ -275,6 +275,26 @@ namespace sd_sample {
         }
     }
 
+    void SampleStepCacheDispatcher::invalidate_condition(const void* condition) {
+        if (condition == nullptr) {
+            return;
+        }
+
+        switch (runtime.mode) {
+            case SampleCacheMode::EASYCACHE:
+                runtime.easycache.cache_diffs.erase(condition);
+                break;
+            case SampleCacheMode::UCACHE:
+                runtime.ucache.cache_diffs.erase(condition);
+                break;
+            case SampleCacheMode::CACHEDIT:
+                runtime.cachedit.cache_diffs.erase(condition);
+                break;
+            case SampleCacheMode::NONE:
+                break;
+        }
+    }
+
     bool SampleStepCacheDispatcher::is_step_skipped() const {
         switch (runtime.mode) {
             case SampleCacheMode::EASYCACHE:
diff --git a/src/runtime/sample-cache.h b/src/runtime/sample-cache.h
index cb0fe7bc58..30ff66e5fe 100644
--- a/src/runtime/sample-cache.h
+++ b/src/runtime/sample-cache.h
@@ -46,6 +46,7 @@ namespace sd_sample {
 
         bool before_condition(const void* condition, const sd::Tensor<float>& input, sd::Tensor<float>* output);
         void after_condition(const void* condition, const sd::Tensor<float>& input, const sd::Tensor<float>& output);
+        void invalidate_condition(const void* condition);
         bool is_step_skipped() const;
     };
 
diff --git a/src/runtime/spectrum.hpp b/src/runtime/spectrum.hpp
index e30f69308b..1214d53ee1 100644
--- a/src/runtime/spectrum.hpp
+++ b/src/runtime/spectrum.hpp
@@ -74,6 +74,13 @@ struct SpectrumState {
         cnt++;
     }
 
+    // Forget the fitted outputs but keep the step count, so predictions restart from fresh model evaluations.
+    void reset_history() {
+        H_buf.clear();
+        T_buf.clear();
+        num_cached = 0;
+    }
+
     void predict(sd::Tensor<float>* denoised) {
         GGML_ASSERT(denoised != nullptr);
         int64_t F    = (int64_t)H_buf[0].size();
diff --git a/src/runtime/tiling.cpp b/src/runtime/tiling.cpp
index f0a9dbc73b..12c2d6c40b 100644
--- a/src/runtime/tiling.cpp
+++ b/src/runtime/tiling.cpp
@@ -1,61 +1,144 @@
 #include "runtime/tiling.h"
 
 #include <algorithm>
+#include <climits>
 #include <cstdint>
+#include <cstdlib>
+#include <exception>
+#include <memory>
+#include <system_error>
+#include <thread>
 #include <utility>
 #include <vector>
 
 #include "core/util.h"
 #include "ggml.h"
 
-static void sd_tiling_calc_tiles(int& num_tiles_dim,
-                                 float& tile_overlap_factor_dim,
-                                 int small_dim,
-                                 int tile_size,
-                                 const float tile_overlap_factor,
-                                 bool circular) {
-    int tile_overlap     = static_cast<int>(tile_size * tile_overlap_factor);
-    int non_tile_overlap = tile_size - tile_overlap;
+struct TileSpan {
+    int offset;
+    int overlap_before;
+    int overlap_after;
+};
 
-    if (circular) {
-        // circular means the last and first tile are overlapping (wraping around)
-        num_tiles_dim = small_dim / non_tile_overlap;
+static std::vector<TileSpan> sd_tiling_plan_axis(int dimension,
+                                                 int tile_size,
+                                                 float target_overlap,
+                                                 bool circular) {
+    if (tile_size >= dimension) {
+        return {{0, 0, 0}};
+    }
 
-        if (num_tiles_dim < 1) {
-            num_tiles_dim = 1;
-        }
+    const int extent = circular ? dimension : dimension - tile_size;
+    int intervals    = extent;
+    if (tile_size > 1) {
+        const float target = extent / ((1.0f - target_overlap) * tile_size);
+        // Adjacent tiles must overlap, but three tiles must not cover the same point.
+        const int min_intervals = 1 + (extent - 1) / (tile_size - 1);
+        const int max_intervals = std::max(1, static_cast<int>(2LL * extent / tile_size));
+        const int lower         = std::clamp(static_cast<int>(std::floor(target)), min_intervals, max_intervals);
+        const int upper         = std::clamp(static_cast<int>(std::ceil(target)), min_intervals, max_intervals);
+        auto error              = [&](int count) {
+            return std::abs(1.0f / count - 1.0f / target);
+        };
+        intervals = error(upper) < error(lower) ? upper : lower;
+    }
 
-        tile_overlap_factor_dim = (tile_size - small_dim / num_tiles_dim) / (float)tile_size;
+    const int num_tiles = circular ? intervals : intervals + 1;
+    auto position       = [&](int index) -> int {
+        return static_cast<int>(int64_t(index) * extent / intervals);
+    };
+    std::vector<TileSpan> tiles;
+    tiles.reserve(num_tiles);
+    for (int i = 0; i < num_tiles; ++i) {
+        const int offset = position(i);
+        // Keep wrapped neighbors in the same unwrapped coordinate space.
+        const int previous = i > 0 ? position(i - 1) : position(num_tiles - 1) - dimension;
+        const int next     = i + 1 < num_tiles ? position(i + 1) : dimension;
+        const int before   = i > 0 || circular ? previous + tile_size - offset : 0;
+        const int after    = i + 1 < num_tiles || circular ? offset + tile_size - next : 0;
+        tiles.push_back({offset, before, after});
+    }
+    return tiles;
+}
 
-        // if single tile and tile_overlap_factor is not 0, add one to ensure we have at least two overlapping tiles
-        if (num_tiles_dim == 1 && tile_overlap_factor_dim > 0) {
-            num_tiles_dim++;
-            tile_overlap_factor_dim = 0.5;
+int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor, int max_tile_size) {
+    // Overlaps of 1-2 latents (32-latent tiles on a 62 axis) show as a seam; require >= half the target and >= 2.
+    const int min_tile_size = 4;
+    const int limit         = max_tile_size > 0 ? std::max(max_tile_size, min_tile_size) : INT_MAX;
+    tile_size               = std::min(tile_size, limit);
+    if (tile_overlap_factor <= 0.f || tile_size >= small_dim) {
+        return tile_size;
+    }
+    auto overlaps_enough = [&](int size, bool replacement) {
+        int min_overlap = std::max(2, static_cast<int>(size * tile_overlap_factor * 0.5f));
+        if (replacement) {
+            // Half the target still leaves a line on 16x VAEs (Qwen-Image-2.1 at 480 px); 8 latents does not.
+            min_overlap = std::max(min_overlap, std::min(8, static_cast<int>(size * tile_overlap_factor)));
         }
-
-        return;
+        const auto tiles = sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false);
+        for (size_t i = 0; i + 1 < tiles.size(); ++i) {
+            if (tiles[i].overlap_after < min_overlap) {
+                return false;
+            }
+        }
+        return true;
+    };
+    if (overlaps_enough(tile_size, false)) {
+        return tile_size;
     }
-    // else, non-circular means the last and first tile are not overlapping
-
-    num_tiles_dim     = (small_dim - tile_overlap) / non_tile_overlap;
-    int overshoot_dim = ((num_tiles_dim + 1) * non_tile_overlap + tile_overlap) % small_dim;
-
-    if ((overshoot_dim != non_tile_overlap) && (overshoot_dim <= num_tiles_dim * (tile_size / 2 - tile_overlap))) {
-        // if tiles don't fit perfectly using the desired overlap
-        // and there is enough room to squeeze an extra tile without overlap becoming >0.5
-        num_tiles_dim++;
+    auto tile_count = [&](int size) {
+        return static_cast<int>(sd_tiling_plan_axis(small_dim, size, tile_overlap_factor, false).size());
+    };
+    // The nearest passing size (smaller first, then larger up to 2x) caps the tile count so tiny tiles never win.
+    int nearest = 0;
+    for (int size = tile_size - 1; size >= min_tile_size && nearest == 0; --size) {
+        if (overlaps_enough(size, false)) {
+            nearest = size;
+        }
     }
-
-    tile_overlap_factor_dim = (float)(tile_size * num_tiles_dim - small_dim) / (float)(tile_size * (num_tiles_dim - 1));
-    if (num_tiles_dim <= 2) {
-        if (small_dim <= tile_size) {
-            num_tiles_dim           = 1;
-            tile_overlap_factor_dim = 0;
-        } else {
-            num_tiles_dim           = 2;
-            tile_overlap_factor_dim = (2 * tile_size - small_dim) / (float)tile_size;
+    for (int size = tile_size + 1; nearest == 0 && size < small_dim && size <= std::min(2 * tile_size, limit); ++size) {
+        if (overlaps_enough(size, false)) {
+            nearest = size;
         }
     }
+    if (nearest == 0) {
+        return tile_size;
+    }
+    const int max_tiles = tile_count(nearest);
+    auto pick = [&](int lowest, int highest, bool replacement) {
+        int best_size     = 0;
+        int64_t best_cost = 0;
+        int best_tiles    = 0;
+        for (int size = lowest; size <= highest; ++size) {
+            if (!overlaps_enough(size, replacement)) {
+                continue;
+            }
+            const int tiles = tile_count(size);
+            if (tiles > max_tiles) {
+                continue;
+            }
+            const int64_t cost = static_cast<int64_t>(tiles) * size;
+            const bool better = best_size == 0 || cost < best_cost || (cost == best_cost && tiles < best_tiles);
+            if (better) {
+                best_size  = size;
+                best_cost  = cost;
+                best_tiles = tiles;
+            }
+        }
+        return best_size;
+    };
+    const int capped = std::min({small_dim - 1, tile_size + tile_size / 4, limit});
+    const int widest = std::min({small_dim - 1, 2 * tile_size, limit});
+    for (bool replacement : {true, false}) {
+        int size = pick(min_tile_size, capped, replacement);
+        if (size == 0) {
+            size = pick(capped + 1, widest, replacement);
+        }
+        if (size != 0) {
+            return size;
+        }
+    }
+    return nearest;
 }
 
 static int64_t sd_tensor_plane_size(const sd::Tensor<float>& tensor) {
@@ -74,194 +157,311 @@ static sd::Tensor<float> sd_tensor_split_2d(const sd::Tensor<float>& input, int
     int64_t input_plane  = sd_tensor_plane_size(input);
     int64_t output_plane = sd_tensor_plane_size(output);
     int64_t plane_count  = input.numel() / input_plane;
-    for (int iy = 0; iy < height; iy++) {
-        for (int ix = 0; ix < width; ix++) {
-            int64_t src_xy = (ix + x) % input_width + input_width * ((iy + y) % input_height);
-            int64_t dst_xy = ix + width * iy;
-            for (int64_t plane = 0; plane < plane_count; ++plane) {
-                output[plane * output_plane + dst_xy] = input[plane * input_plane + src_xy];
+    // Plane-outer: planes are contiguous, so the inner loop streams instead of striding a plane per element.
+    std::vector<int64_t> src_x(static_cast<size_t>(width));
+    for (int ix = 0; ix < width; ix++) {
+        src_x[ix] = (ix + x) % input_width;
+    }
+    for (int64_t plane = 0; plane < plane_count; ++plane) {
+        const float* src = input.data() + plane * input_plane;
+        float* dst       = output.data() + plane * output_plane;
+        for (int iy = 0; iy < height; iy++) {
+            const float* src_row = src + input_width * ((iy + y) % input_height);
+            float* dst_row       = dst + static_cast<int64_t>(width) * iy;
+            for (int ix = 0; ix < width; ix++) {
+                dst_row[ix] = src_row[src_x[ix]];
             }
         }
     }
     return output;
 }
 
-static void sd_tensor_merge_2d(const sd::Tensor<float>& input,
+template <typename Fn>
+static void sd_parallel_for_planes(int64_t plane_count, int64_t plane_elements, Fn&& fn) {
+    // Planes are independent and each element is still produced by one thread with the same
+    // arithmetic, so splitting by plane does not change the result.
+    const int64_t min_elements = 1 << 18;
+    int64_t threads            = std::min<int64_t>(plane_count, std::max<int64_t>(1, plane_count * plane_elements / min_elements));
+    threads                    = std::min<int64_t>(threads, std::max(1u, std::min(16u, std::thread::hardware_concurrency())));
+    if (threads <= 1) {
+        fn(0, plane_count);
+        return;
+    }
+    std::vector<std::thread> workers;
+    const int64_t per_thread = (plane_count + threads - 1) / threads;
+    for (int64_t begin = per_thread; begin < plane_count; begin += per_thread) {
+        workers.emplace_back([&fn, begin, end = std::min(plane_count, begin + per_thread)]() { fn(begin, end); });
+    }
+    fn(0, std::min(plane_count, per_thread));
+    for (auto& worker : workers) {
+        worker.join();
+    }
+}
+
+static void sd_tensor_merge_2d(const float* input_data,
+                               int64_t in_width,
+                               int64_t in_height,
+                               int64_t plane_count,
                                sd::Tensor<float>* output,
                                int x,
                                int y,
-                               int overlap_x,
-                               int overlap_y,
-                               bool circular_x,
-                               bool circular_y,
-                               int x_skip,
-                               int y_skip) {
+                               int overlap_left,
+                               int overlap_right,
+                               int overlap_top,
+                               int overlap_bottom) {
     GGML_ASSERT(output != nullptr);
-    int64_t width        = input.shape()[0];
-    int64_t height       = input.shape()[1];
-    int64_t img_width    = output->shape()[0];
-    int64_t img_height   = output->shape()[1];
-    int64_t input_plane  = sd_tensor_plane_size(input);
-    int64_t output_plane = sd_tensor_plane_size(*output);
-    int64_t plane_count  = input.numel() / input_plane;
-    GGML_ASSERT(output->numel() / output_plane == plane_count);
 
-    // unclamped -> expects x in the range [0-1]
+    int64_t out_width  = output->shape()[0];
+    int64_t out_height = output->shape()[1];
+    int64_t in_size    = in_width * in_height;
+    int64_t out_size   = sd_tensor_plane_size(*output);
+
+    GGML_ASSERT(output->numel() == plane_count * out_size);
+    GGML_ASSERT(x >= 0 && y >= 0);
+    GGML_ASSERT(x < out_width && in_width <= out_width);
+    GGML_ASSERT(y < out_height && in_height <= out_height);
+    GGML_ASSERT(overlap_left >= 0 && overlap_right >= 0);
+    GGML_ASSERT(overlap_top >= 0 && overlap_bottom >= 0);
+
     auto smootherstep_f32 = [](const float x) -> float {
-        GGML_ASSERT(x >= 0.f && x <= 1.f);
         return x * x * x * (x * (6.0f * x - 15.0f) + 10.0f);
     };
-
-    for (int iy = y_skip; iy < height; iy++) {
-        for (int ix = x_skip; ix < width; ix++) {
-            int64_t src_xy = ix + width * iy;
-            int64_t ox     = (x + ix) % img_width;
-            int64_t oy     = (y + iy) % img_height;
-            int64_t dst_xy = ox + img_width * oy;
-            for (int64_t plane = 0; plane < plane_count; ++plane) {
-                float new_value = input[plane * input_plane + src_xy];
-                if (overlap_x > 0 || overlap_y > 0) {
-                    float old_value   = (*output)[plane * output_plane + dst_xy];
-                    const float x_f_0 = (circular_x || (overlap_x > 0 && x > 0)) ? (ix - x_skip) / float(overlap_x) : 1.f;
-                    const float x_f_1 = (circular_x || (overlap_x > 0 && x < (img_width - width))) ? (width - ix) / float(overlap_x) : 1.f;
-                    const float y_f_0 = (circular_y || (overlap_y > 0 && y > 0)) ? (iy - y_skip) / float(overlap_y) : 1.f;
-                    const float y_f_1 = (circular_y || (overlap_y > 0 && y < (img_height - height))) ? (height - iy) / float(overlap_y) : 1.f;
-                    const float x_f   = std::min(std::min(x_f_0, x_f_1), 1.f);
-                    const float y_f   = std::min(std::min(y_f_0, y_f_1), 1.f);
-                    (*output)[plane * output_plane + dst_xy] =
-                        old_value + new_value * smootherstep_f32(y_f) * smootherstep_f32(x_f);
-                } else {
-                    (*output)[plane * output_plane + dst_xy] = new_value;
+    // Weights depend only on the column or row; the per-element arithmetic order is unchanged.
+    std::vector<float> x_weights(static_cast<size_t>(in_width));
+    std::vector<float> y_weights(static_cast<size_t>(in_height));
+    std::vector<int64_t> dst_x(static_cast<size_t>(in_width));
+    for (int ix = 0; ix < in_width; ++ix) {
+        float x_f = 1.0f;
+        if (ix < overlap_left) {
+            x_f = static_cast<float>(ix) / overlap_left;
+        }
+        if (ix >= in_width - overlap_right) {
+            x_f = static_cast<float>(in_width - ix) / overlap_right;
+        }
+        x_weights[ix] = smootherstep_f32(std::clamp(x_f, 0.0f, 1.0f));
+        dst_x[ix]     = (x + ix) % out_width;
+    }
+    for (int iy = 0; iy < in_height; ++iy) {
+        float y_f = 1.0f;
+        if (iy < overlap_top) {
+            y_f = static_cast<float>(iy) / overlap_top;
+        }
+        if (iy >= in_height - overlap_bottom) {
+            y_f = static_cast<float>(in_height - iy) / overlap_bottom;
+        }
+        y_weights[iy] = smootherstep_f32(std::clamp(y_f, 0.0f, 1.0f));
+    }
+    float* output_data = output->data();
+    sd_parallel_for_planes(plane_count, in_size, [&](int64_t first_plane, int64_t last_plane) {
+        for (int64_t plane = first_plane; plane < last_plane; ++plane) {
+            const float* src = input_data + plane * in_size;
+            float* dst       = output_data + plane * out_size;
+            for (int iy = 0; iy < in_height; ++iy) {
+                const float y_weight = y_weights[iy];
+                const float* src_row = src + in_width * iy;
+                float* dst_row       = dst + out_width * ((y + iy) % out_height);
+                for (int ix = 0; ix < in_width; ++ix) {
+                    dst_row[dst_x[ix]] += x_weights[ix] * y_weight * src_row[ix];
                 }
             }
         }
-    }
+    });
 }
 
 sd::Tensor<float> process_tiles_2d(const sd::Tensor<float>& input,
                                    int output_width,
                                    int output_height,
                                    int scale,
-                                   int p_tile_size_x,
-                                   int p_tile_size_y,
+                                   int p_tile_size_w,
+                                   int p_tile_size_h,
                                    float tile_overlap_factor,
                                    bool circular_x,
                                    bool circular_y,
                                    const TileProcessCallback& on_processing,
                                    bool silent) {
+    return process_tiles_2d_batched(
+        input,
+        output_width,
+        output_height,
+        scale,
+        p_tile_size_w,
+        p_tile_size_h,
+        tile_overlap_factor,
+        circular_x,
+        circular_y,
+        [](int) { return 1; },
+        [&](const std::vector<sd::Tensor<float>>& tiles) {
+            TileBatchOutput output;
+            output.data = on_processing(tiles[0]);
+            if (!output.data.empty()) {
+                output.stack_dim = static_cast<size_t>(output.data.dim() - 1);
+            }
+            return output;
+        },
+        silent);
+}
+
+sd::Tensor<float> process_tiles_2d_batched(const sd::Tensor<float>& input,
+                                           int output_width,
+                                           int output_height,
+                                           int scale,
+                                           int p_tile_size_w,
+                                           int p_tile_size_h,
+                                           float tile_overlap_factor,
+                                           bool circular_x,
+                                           bool circular_y,
+                                           const TileBatchSizeCallback& batch_size,
+                                           const TileBatchProcessCallback& on_processing,
+                                           bool silent) {
     sd::Tensor<float> output;
     int input_width  = static_cast<int>(input.shape()[0]);
     int input_height = static_cast<int>(input.shape()[1]);
 
-    GGML_ASSERT(((input_width / output_width) == (input_height / output_height)) &&
-                ((output_width / input_width) == (output_height / input_height)));
     GGML_ASSERT(((input_width / output_width) == scale) ||
                 ((output_width / input_width) == scale));
 
-    int small_width  = output_width;
-    int small_height = output_height;
-    bool decode      = output_width > input_width;
-    if (decode) {
-        small_width  = input_width;
-        small_height = input_height;
-    }
+    bool decode      = output_width > input_width;  // scale up
+    int small_width  = decode ? input_width : output_width;
+    int small_height = decode ? input_height : output_height;
+    int scale_in     = decode ? 1 : scale;
+    int scale_out    = decode ? scale : 1;
 
-    int num_tiles_x;
-    float tile_overlap_factor_x;
-    sd_tiling_calc_tiles(num_tiles_x, tile_overlap_factor_x, small_width, p_tile_size_x, tile_overlap_factor, circular_x);
+    const auto tiles_x = sd_tiling_plan_axis(small_width, p_tile_size_w, tile_overlap_factor, circular_x);
+    const auto tiles_y = sd_tiling_plan_axis(small_height, p_tile_size_h, tile_overlap_factor, circular_y);
 
-    int num_tiles_y;
-    float tile_overlap_factor_y;
-    sd_tiling_calc_tiles(num_tiles_y, tile_overlap_factor_y, small_height, p_tile_size_y, tile_overlap_factor, circular_y);
+    int tile_width         = std::min(p_tile_size_w, small_width);
+    int tile_height        = std::min(p_tile_size_h, small_height);
+    int input_tile_width   = tile_width * scale_in;
+    int input_tile_height  = tile_height * scale_in;
+    int output_tile_width  = tile_width * scale_out;
+    int output_tile_height = tile_height * scale_out;
 
-    int tile_overlap_x     = static_cast<int32_t>(p_tile_size_x * tile_overlap_factor_x);
-    int non_tile_overlap_x = p_tile_size_x - tile_overlap_x;
-    int tile_overlap_y     = static_cast<int32_t>(p_tile_size_y * tile_overlap_factor_y);
-    int non_tile_overlap_y = p_tile_size_y - tile_overlap_y;
-    int tile_size_x        = p_tile_size_x < small_width ? p_tile_size_x : small_width;
-    int tile_size_y        = p_tile_size_y < small_height ? p_tile_size_y : small_height;
-    int input_tile_size_x  = tile_size_x;
-    int input_tile_size_y  = tile_size_y;
-    int output_tile_size_x = tile_size_x;
-    int output_tile_size_y = tile_size_y;
-    if (decode) {
-        output_tile_size_x *= scale;
-        output_tile_size_y *= scale;
-    } else {
-        input_tile_size_x *= scale;
-        input_tile_size_y *= scale;
+    struct TilePlacement {
+        const TileSpan* x;
+        const TileSpan* y;
+    };
+    std::vector<TilePlacement> placements;
+    for (const auto& y : tiles_y) {
+        for (const auto& x : tiles_x) {
+            placements.push_back({&x, &y});
+        }
     }
 
-    int num_tiles   = num_tiles_x * num_tiles_y;
-    int tile_count  = 1;
-    bool last_y     = false;
-    bool last_x     = false;
-    float last_time = 0.0f;
+    const int num_tiles = static_cast<int>(placements.size());
+    int tile_count      = 0;
     if (!silent) {
-        LOG_VERBOSE("num tiles : %d, %d ", num_tiles_x, num_tiles_y);
-        LOG_VERBOSE("optimal overlap : %f, %f (targeting %f)", tile_overlap_factor_x, tile_overlap_factor_y, tile_overlap_factor);
+        LOG_VERBOSE("num tiles : %d, %d ", static_cast<int>(tiles_x.size()), static_cast<int>(tiles_y.size()));
         LOG_VERBOSE("processing %i tiles", num_tiles);
         pretty_progress(0, num_tiles, 0.0f);
     }
-    for (int y = 0; y < small_height && !last_y; y += non_tile_overlap_y) {
-        int dy = 0;
-        if (!circular_y && y + tile_size_y >= small_height) {
-            int original_y = y;
-            y              = small_height - tile_size_y;
-            dy             = original_y - y;
-            if (decode) {
-                dy *= scale;
-            }
-            last_y = true;
+    // Blend on a worker while the next batch computes, still in tile order; SD_TILE_ASYNC_MERGE=0 blends inline.
+    static const bool async_merge = []() {
+        const char* value = getenv("SD_TILE_ASYNC_MERGE");
+        return value == nullptr || value[0] == '\0' || atoi(value) != 0;
+    }();
+    std::thread merger;
+    bool merge_failed = false;
+    auto join_merger  = [&]() {
+        if (merger.joinable()) {
+            merger.join();
         }
-        for (int x = 0; x < small_width && !last_x; x += non_tile_overlap_x) {
-            int dx = 0;
-            if (!circular_x && x + tile_size_x >= small_width) {
-                int original_x = x;
-                x              = small_width - tile_size_x;
-                dx             = original_x - x;
-                if (decode) {
-                    dx *= scale;
-                }
-                last_x = true;
-            }
-
-            int x_in  = decode ? x : scale * x;
-            int y_in  = decode ? y : scale * y;
-            int x_out = decode ? x * scale : x;
-            int y_out = decode ? y * scale : y;
-
-            int overlap_x_out = decode ? tile_overlap_x * scale : tile_overlap_x;
-            int overlap_y_out = decode ? tile_overlap_y * scale : tile_overlap_y;
-
-            int64_t t1       = ggml_time_ms();
-            auto input_tile  = sd_tensor_split_2d(input, input_tile_size_x, input_tile_size_y, x_in, y_in);
-            auto output_tile = on_processing(input_tile);
-            if (output_tile.empty()) {
-                return {};
+    };
+    struct MergerGuard {
+        std::thread& thread;
+        ~MergerGuard() {
+            if (thread.joinable()) {
+                thread.join();
             }
-            GGML_ASSERT(output_tile.shape()[0] == output_tile_size_x && output_tile.shape()[1] == output_tile_size_y);
+        }
+    } merger_guard{merger};
+    std::vector<sd::Tensor<float>> input_tiles;
+    for (size_t first = 0; first < placements.size();) {
+        const int remaining = static_cast<int>(placements.size() - first);
+        const int count     = std::max(1, std::min(remaining, batch_size(remaining)));
+        const size_t last   = first + static_cast<size_t>(count);
+        int64_t t1          = ggml_time_ms();
+        input_tiles.clear();
+        for (size_t i = first; i < last; ++i) {
+            input_tiles.push_back(sd_tensor_split_2d(input,
+                                                     input_tile_width,
+                                                     input_tile_height,
+                                                     placements[i].x->offset * scale_in,
+                                                     placements[i].y->offset * scale_in));
+        }
+        auto batch = on_processing(input_tiles);
+        if (batch.data.empty()) {
+            return {};
+        }
+        const int64_t batch_tiles = static_cast<int64_t>(last - first);
+        GGML_ASSERT(batch.stack_dim < static_cast<size_t>(batch.data.dim()) && batch.data.shape()[batch.stack_dim] % batch_tiles == 0);
+        for (size_t d = batch.stack_dim + 1; d < static_cast<size_t>(batch.data.dim()); ++d) {
+            GGML_ASSERT(batch.data.shape()[d] == 1);
+        }
+        std::vector<int64_t> tile_shape = batch.data.shape();
+        tile_shape[batch.stack_dim] /= batch_tiles;
+        GGML_ASSERT(tile_shape[0] == output_tile_width && tile_shape[1] == output_tile_height);
+        auto merge_batch = [&, first, last, tile_shape](const sd::Tensor<float>& data) {
+            const int64_t tile_numel  = data.numel() / static_cast<int64_t>(last - first);
+            const int64_t plane_count = tile_numel / (tile_shape[0] * tile_shape[1]);
             if (output.empty()) {
-                std::vector<int64_t> output_shape = output_tile.shape();
+                std::vector<int64_t> output_shape = tile_shape;
                 output_shape[0]                   = output_width;
                 output_shape[1]                   = output_height;
                 output                            = sd::Tensor<float>::zeros(std::move(output_shape));
             }
-            sd_tensor_merge_2d(output_tile, &output, x_out, y_out, overlap_x_out, overlap_y_out, circular_x, circular_y, dx, dy);
+            for (size_t i = first; i < last; ++i) {
+                const auto& x = *placements[i].x;
+                const auto& y = *placements[i].y;
+                sd_tensor_merge_2d(data.data() + static_cast<int64_t>(i - first) * tile_numel,
+                                   tile_shape[0],
+                                   tile_shape[1],
+                                   plane_count,
+                                   &output,
+                                   x.offset * scale_out,
+                                   y.offset * scale_out,
+                                   x.overlap_before * scale_out,
+                                   x.overlap_after * scale_out,
+                                   y.overlap_before * scale_out,
+                                   y.overlap_after * scale_out);
+            }
+        };
+        join_merger();
+        if (merge_failed) {
+            return {};
+        }
+        bool merged_async = false;
+        if (async_merge && last < placements.size()) {
+            auto data = std::make_shared<sd::Tensor<float>>(std::move(batch.data));
+            try {
+                merger = std::thread([&merge_failed, merge_batch, data]() {
+                    try {
+                        merge_batch(*data);
+                    } catch (const std::exception& error) {
+                        LOG_ERROR("tile merge failed: %s", error.what());
+                        merge_failed = true;
+                    }
+                });
+                merged_async = true;
+            } catch (const std::system_error&) {
+                merge_batch(*data);  // no thread available: merge inline
+                merged_async = true;
+            }
+        }
+        if (!merged_async) {
+            merge_batch(batch.data);
+        }
 
+        const float tile_time = (ggml_time_ms() - t1) / 1000.0f / static_cast<float>(last - first);
+        for (size_t i = first; i < last; ++i) {
+            ++tile_count;
             if (!silent) {
-                int64_t t2 = ggml_time_ms();
-                last_time  = (t2 - t1) / 1000.0f;
-                pretty_progress(tile_count, num_tiles, last_time);
+                pretty_progress(tile_count, num_tiles, tile_time);
             }
-            tile_count++;
         }
-        last_x = false;
-    }
-    if (!silent && tile_count < num_tiles) {
-        pretty_progress(num_tiles, num_tiles, last_time);
+        first = last;
     }
-    if (output.empty()) {
+    join_merger();
+    if (merge_failed) {
         return {};
     }
     return output;
diff --git a/src/runtime/tiling.h b/src/runtime/tiling.h
index a832a644e3..029078fd30 100644
--- a/src/runtime/tiling.h
+++ b/src/runtime/tiling.h
@@ -2,21 +2,49 @@
 #define __SD_RUNTIME_TILING_H__
 
 #include <functional>
+#include <vector>
 
 #include "core/tensor.hpp"
 
-using TileProcessCallback = std::function<sd::Tensor<float>(const sd::Tensor<float>&)>;
+using TileProcessCallback   = std::function<sd::Tensor<float>(const sd::Tensor<float>&)>;
+using TileBatchSizeCallback = std::function<int(int remaining_tiles)>;
+// Output tiles of one batch stacked along stack_dim, which must be the highest axis with an extent
+// above 1, so tile i is the i-th contiguous block of data. Each tile's shape is data.shape() with
+// shape[stack_dim] divided by the number of tiles.
+struct TileBatchOutput {
+    sd::Tensor<float> data;
+    size_t stack_dim = 0;
+};
+using TileBatchProcessCallback = std::function<TileBatchOutput(const std::vector<sd::Tensor<float>>&)>;
 
 sd::Tensor<float> process_tiles_2d(const sd::Tensor<float>& input,
                                    int output_width,
                                    int output_height,
                                    int scale,
-                                   int p_tile_size_x,
-                                   int p_tile_size_y,
+                                   int p_tile_size_w,
+                                   int p_tile_size_h,
                                    float tile_overlap_factor,
                                    bool circular_x,
                                    bool circular_y,
                                    const TileProcessCallback& on_processing,
                                    bool silent = false);
 
+// Same tile plan and merge order as process_tiles_2d, but hands up to batch_size(remaining)
+// consecutive tiles to the callback at once.
+sd::Tensor<float> process_tiles_2d_batched(const sd::Tensor<float>& input,
+                                           int output_width,
+                                           int output_height,
+                                           int scale,
+                                           int p_tile_size_w,
+                                           int p_tile_size_h,
+                                           float tile_overlap_factor,
+                                           bool circular_x,
+                                           bool circular_y,
+                                           const TileBatchSizeCallback& batch_size,
+                                           const TileBatchProcessCallback& on_processing,
+                                           bool silent = false);
+
+// tile_size adjusted so tiles never barely overlap; max_tile_size > 0 (OOM retry) caps it, never below 4.
+int sd_tiling_seam_safe_tile_size(int small_dim, int tile_size, float tile_overlap_factor, int max_tile_size = 0);
+
 #endif  // __SD_RUNTIME_TILING_H__
diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp
index a8f83336d9..d476f92d6b 100644
--- a/src/stable-diffusion.cpp
+++ b/src/stable-diffusion.cpp
@@ -326,6 +326,7 @@ void sd_hires_params_init(sd_hires_params_t* hires_params) {
 void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) {
     *sd_ctx_params                           = {};
     sd_ctx_params->n_threads                 = sd_get_num_physical_cores();
+    sd_ctx_params->conditioning_cache_size   = 4;
     sd_ctx_params->wtype                     = SD_TYPE_COUNT;
     sd_ctx_params->rng_type                  = CUDA_RNG;
     sd_ctx_params->sampler_rng_type          = RNG_TYPE_COUNT;
@@ -334,8 +335,11 @@ void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) {
     sd_ctx_params->max_vram                  = nullptr;
     sd_ctx_params->disable_prefetch          = false;
     sd_ctx_params->disable_segmented_compute = false;
+    sd_ctx_params->batched_cfg               = true;
     sd_ctx_params->eager_load                = false;
     sd_ctx_params->enable_mmap               = false;
+    sd_ctx_params->flash_attn_mode           = nullptr;
+    sd_ctx_params->conv_direct_mode          = nullptr;
     sd_ctx_params->diffusion_flash_attn      = false;
     sd_ctx_params->sage_attn                 = false;
     sd_ctx_params->linear_scale              = 0.f;
@@ -378,6 +382,7 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
              "pulid_weights_path: %s\n"
              "tensor_type_rules: %s\n"
              "n_threads: %d\n"
+             "conditioning_cache_size: %d\n"
              "wtype: %s\n"
              "rng_type: %s\n"
              "sampler_rng_type: %s\n"
@@ -392,6 +397,8 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
              "model_args: %s\n"
              "auto_fit: %s\n"
              "flash_attn: %s\n"
+             "flash_attn_mode: %s\n"
+             "conv_direct_mode: %s\n"
              "diffusion_flash_attn: %s\n"
              "sage_attn: %s\n"
              "linear_scale: %g\n"
@@ -418,6 +425,7 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
              SAFE_STR(sd_ctx_params->pulid_weights_path),
              SAFE_STR(sd_ctx_params->tensor_type_rules),
              sd_ctx_params->n_threads,
+             sd_ctx_params->conditioning_cache_size,
              sd_type_name(sd_ctx_params->wtype),
              sd_rng_type_name(sd_ctx_params->rng_type),
              sd_rng_type_name(sd_ctx_params->sampler_rng_type),
@@ -432,6 +440,8 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
              SAFE_STR(sd_ctx_params->model_args),
              BOOL_STR(sd_ctx_params->auto_fit),
              BOOL_STR(sd_ctx_params->flash_attn),
+             sd_ctx_params->flash_attn_mode && sd_ctx_params->flash_attn_mode[0] ? sd_ctx_params->flash_attn_mode : "auto (legacy flags honored)",
+             sd_ctx_params->conv_direct_mode && sd_ctx_params->conv_direct_mode[0] ? sd_ctx_params->conv_direct_mode : "auto (legacy flags honored)",
              BOOL_STR(sd_ctx_params->diffusion_flash_attn),
              BOOL_STR(sd_ctx_params->sage_attn),
              sd_ctx_params->linear_scale,
@@ -630,14 +640,6 @@ struct sd_ctx_t {
     StableDiffusionGGML* sd = nullptr;
 };
 
-static bool sd_version_supports_video_generation(SDVersion version) {
-    return version == VERSION_SVD || sd_version_is_wan(version) || sd_version_is_hunyuan_video(version) || sd_version_is_lingbot_video(version) || sd_version_is_ltxav(version) || sd_version_is_minimax_h3(version);
-}
-
-static bool sd_version_supports_image_generation(SDVersion version) {
-    return !sd_version_supports_video_generation(version);
-}
-
 sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params) {
     sd_ctx_t* sd_ctx = (sd_ctx_t*)malloc(sizeof(sd_ctx_t));
     if (sd_ctx == nullptr) {
@@ -702,6 +704,13 @@ SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) {
     return sd_version_supports_video_generation(sd_ctx->sd->version);
 }
 
+SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx) {
+    if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+        return false;
+    }
+    return sd_version_supports_ref_image_conditioning(sd_ctx->sd->version);
+}
+
 SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) {
     if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) {
         return false;
@@ -759,28 +768,20 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
                            int* num_frames_out,
                            sd_audio_t** audio_out,
                            int* fps_out) {
-    if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_vid_gen_params == nullptr) {
-        if (fps_out != nullptr) {
-            *fps_out = 0;
-        }
-        return false;
-    }
-
-    if (frames_out != nullptr) {
+    if (frames_out != nullptr)
         *frames_out = nullptr;
-    }
-    if (audio_out != nullptr) {
+    if (audio_out != nullptr)
         *audio_out = nullptr;
-    }
-    if (num_frames_out != nullptr) {
+    if (num_frames_out != nullptr)
         *num_frames_out = 0;
+    if (fps_out != nullptr)
+        *fps_out = 0;
+    if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_vid_gen_params == nullptr) {
+        return false;
     }
 
     StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd);
     if (!execution.ready) {
-        if (fps_out != nullptr) {
-            *fps_out = 0;
-        }
         return false;
     }
 
diff --git a/src/tokenizers/bpe_tokenizer.cpp b/src/tokenizers/bpe_tokenizer.cpp
index 79b1029e01..ca86d65db1 100644
--- a/src/tokenizers/bpe_tokenizer.cpp
+++ b/src/tokenizers/bpe_tokenizer.cpp
@@ -242,7 +242,7 @@ bool BPETokenizer::encode(const std::string& text, std::vector<int>& result, on_
         ss << "\"" << token << "\", ";
     }
     ss << "]";
-    LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str());
+    LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", escape_newlines(text).c_str(), bpe_tokens.size(), escape_newlines(ss.str()).c_str());
     result = std::move(bpe_tokens);
     return true;
 }
diff --git a/src/tokenizers/clip_tokenizer.cpp b/src/tokenizers/clip_tokenizer.cpp
index 781104aee0..843790bd55 100644
--- a/src/tokenizers/clip_tokenizer.cpp
+++ b/src/tokenizers/clip_tokenizer.cpp
@@ -63,7 +63,7 @@ void CLIPTokenizer::load_from_merges(const std::string& merges_utf8_str) {
     }
     vocab.push_back(utf8_to_utf32("<|startoftext|>"));
     vocab.push_back(utf8_to_utf32("<|endoftext|>"));
-    LOG_VERBOSE("vocab size: %zu", vocab.size());
+    LOG_DEBUG("vocab size: %zu", vocab.size());
     int i = 0;
     for (const auto& token : vocab) {
         encoder[token] = i;
diff --git a/src/tokenizers/gemma_tokenizer.cpp b/src/tokenizers/gemma_tokenizer.cpp
index f454f0a6a8..8fa9c895e5 100644
--- a/src/tokenizers/gemma_tokenizer.cpp
+++ b/src/tokenizers/gemma_tokenizer.cpp
@@ -29,7 +29,7 @@ void GemmaTokenizer::load_from_merges(const std::string& merges_utf8_str, const
         decoder[i]           = token;
     }
     encoder_len = static_cast<int>(vocab.size());
-    LOG_VERBOSE("vocab size: %d", encoder_len);
+    LOG_DEBUG("vocab size: %d", encoder_len);
 
     std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
     std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
@@ -37,7 +37,7 @@ void GemmaTokenizer::load_from_merges(const std::string& merges_utf8_str, const
         size_t space_pos = merge.find(' ');
         merge_pairs.emplace_back(merge.substr(0, space_pos), merge.substr(space_pos + 1));
     }
-    LOG_VERBOSE("merges size %zu", merge_pairs.size());
+    LOG_DEBUG("merges size %zu", merge_pairs.size());
 
     int rank = 0;
     for (const auto& merge : merge_pairs) {
diff --git a/src/tokenizers/hf_tokenizer.cpp b/src/tokenizers/hf_tokenizer.cpp
index 821d41d797..6e64018191 100644
--- a/src/tokenizers/hf_tokenizer.cpp
+++ b/src/tokenizers/hf_tokenizer.cpp
@@ -4,6 +4,7 @@
 #include <array>
 #include <climits>
 #include <cstdlib>
+#include <filesystem>
 #include <fstream>
 #include <queue>
 #include <sstream>
@@ -495,7 +496,7 @@ struct HFTokenizer::Impl {
 
 HFTokenizer::HFTokenizer(const std::string& path)
     : impl_(new Impl) {
-    std::ifstream stream(path, std::ios::binary);
+    std::ifstream stream(std::filesystem::u8path(path), std::ios::binary);
     tokenizer_require(stream.good(), "cannot open " + path);
     TokenizerJSON config;
     stream >> config;
@@ -735,7 +736,7 @@ bool HFTokenizer::encode(const std::string& text, std::vector<int>& tokens, on_n
         }
     }
     ss << "]";
-    LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", text.c_str(), result.size(), ss.str().c_str());
+    LOG_VERBOSE("split prompt \"%s\" to %zu tokens %s", escape_newlines(text).c_str(), result.size(), escape_newlines(ss.str()).c_str());
     tokens = std::move(result);
     return true;
 }
diff --git a/src/tokenizers/mistral_tokenizer.cpp b/src/tokenizers/mistral_tokenizer.cpp
index 4447169669..77c5f42b88 100644
--- a/src/tokenizers/mistral_tokenizer.cpp
+++ b/src/tokenizers/mistral_tokenizer.cpp
@@ -20,7 +20,7 @@ void MistralTokenizer::load_from_merges(const std::string& merges_utf8_str, cons
         decoder[i]           = token;
     }
     encoder_len = static_cast<int>(vocab.size());
-    LOG_VERBOSE("vocab size: %d", encoder_len);
+    LOG_DEBUG("vocab size: %d", encoder_len);
 
     auto byte_unicode_pairs = bytes_to_unicode();
     byte_encoder            = std::map<int, std::u32string>(byte_unicode_pairs.begin(), byte_unicode_pairs.end());
@@ -28,7 +28,7 @@ void MistralTokenizer::load_from_merges(const std::string& merges_utf8_str, cons
         byte_decoder[pair.second] = pair.first;
     }
     std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
-    LOG_VERBOSE("merges size %zu", merges.size());
+    LOG_DEBUG("merges size %zu", merges.size());
     std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
     for (const auto& merge : merges) {
         size_t space_pos = merge.find(' ');
diff --git a/src/tokenizers/qwen2_tokenizer.cpp b/src/tokenizers/qwen2_tokenizer.cpp
index d01c970e8c..a9f340fcbd 100644
--- a/src/tokenizers/qwen2_tokenizer.cpp
+++ b/src/tokenizers/qwen2_tokenizer.cpp
@@ -11,7 +11,7 @@ void Qwen2Tokenizer::load_from_merges(const std::string& merges_utf8_str) {
     }
 
     std::vector<std::u32string> merges = split_utf32(merges_utf8_str);
-    LOG_VERBOSE("merges size %zu", merges.size());
+    LOG_DEBUG("merges size %zu", merges.size());
     std::vector<std::pair<std::u32string, std::u32string>> merge_pairs;
     for (const auto& merge : merges) {
         size_t space_pos = merge.find(' ');
@@ -36,7 +36,7 @@ void Qwen2Tokenizer::load_from_merges(const std::string& merges_utf8_str) {
         i++;
     }
     encoder_len = i;
-    LOG_VERBOSE("vocab size: %d", encoder_len);
+    LOG_DEBUG("vocab size: %d", encoder_len);
 
     int rank = 0;
     for (const auto& merge : merge_pairs) {
diff --git a/src/tokenizers/t5_unigram_tokenizer.cpp b/src/tokenizers/t5_unigram_tokenizer.cpp
index 36526086e9..342dca1a2d 100644
--- a/src/tokenizers/t5_unigram_tokenizer.cpp
+++ b/src/tokenizers/t5_unigram_tokenizer.cpp
@@ -295,7 +295,13 @@ bool T5UniGramTokenizer::encode(const std::string& input, std::vector<int>& resu
     std::vector<int32_t> tokens;
     std::vector<std::string> token_strs;
     std::string normalized = normalize(input);
-    auto splited_texts     = split_with_special_tokens(normalized, special_tokens);
+    if (normalized.empty()) {
+        // HF reference tokenizers emit no pieces for empty input; pad_tokens
+        // still appends EOS so the sequence becomes [EOS] + padding.
+        result = std::move(tokens);
+        return true;
+    }
+    auto splited_texts = split_with_special_tokens(normalized, special_tokens);
     if (splited_texts.empty()) {
         splited_texts.push_back(normalized);  // for empty string
     }
@@ -337,7 +343,7 @@ bool T5UniGramTokenizer::encode(const std::string& input, std::vector<int>& resu
         ss << "\"" << token_str << "\", ";
     }
     ss << "]";
-    LOG_VERBOSE("split prompt \"%s\" to tokens %s", input.c_str(), ss.str().c_str());
+    LOG_VERBOSE("split prompt \"%s\" to tokens %s", escape_newlines(input).c_str(), escape_newlines(ss.str()).c_str());
 
     result = std::move(tokens);
     return true;
diff --git a/src/upscaler.cpp b/src/upscaler.cpp
index 341c76ecfc..dfe5e4b22f 100644
--- a/src/upscaler.cpp
+++ b/src/upscaler.cpp
@@ -25,13 +25,38 @@ UpscalerGGML::~UpscalerGGML() {
     // ModelManager holds raw ggml tensor pointers owned by the runner context.
     model_manager.reset();
     esrgan_upscaler.reset();
+    mosr_upscaler.reset();
 }
 
 void UpscalerGGML::set_max_graph_vram_bytes(size_t max_vram_bytes) {
     max_graph_vram_bytes = max_vram_bytes;
+    if (runner() != nullptr) {
+        runner()->set_max_graph_vram_bytes(max_vram_bytes);
+    }
+}
+
+GGMLRunner* UpscalerGGML::runner() {
+    if (mosr_upscaler) {
+        return mosr_upscaler.get();
+    }
+    return esrgan_upscaler.get();
+}
+
+int UpscalerGGML::get_scale() {
+    if (mosr_upscaler) {
+        return mosr_upscaler->config.scale;
+    }
     if (esrgan_upscaler) {
-        esrgan_upscaler->set_max_graph_vram_bytes(max_vram_bytes);
+        return esrgan_upscaler->config.scale;
+    }
+    return 1;
+}
+
+sd::Tensor<float> UpscalerGGML::compute(const sd::Tensor<float>& input_tensor) {
+    if (mosr_upscaler) {
+        return mosr_upscaler->compute(n_threads, input_tensor);
     }
+    return esrgan_upscaler->compute(n_threads, input_tensor);
 }
 
 bool UpscalerGGML::load_from_file(const std::string& esrgan_path,
@@ -82,20 +107,33 @@ bool UpscalerGGML::load_from_file(const std::string& esrgan_path,
     }
     model_loader.set_wtype_override(model_data_type);
     LOG_INFO("Upscaler weight type: %s", ggml_type_name(model_data_type));
-    esrgan_upscaler = std::make_shared<ESRGAN>(backend_for(SDBackendModule::UPSCALER),
-                                               model_loader.get_tensor_storage_map(),
+    const auto& tensor_storage_map = model_loader.get_tensor_storage_map();
+    std::map<std::string, ggml_tensor*> tensors;
+    if (tensor_storage_map.count("gblocks.0.weight") > 0) {
+        mosr_upscaler = std::make_shared<MoSR>(backend_for(SDBackendModule::UPSCALER),
+                                               tensor_storage_map,
                                                model_manager);
-    if (esrgan_upscaler == nullptr || esrgan_upscaler->rrdb_net == nullptr) {
-        LOG_ERROR("init esrgan model from metadata failed: '%s'", esrgan_path.c_str());
-        return false;
+        if (mosr_upscaler->mosr_net == nullptr) {
+            LOG_ERROR("init mosr model from metadata failed: '%s'", esrgan_path.c_str());
+            return false;
+        }
+        mosr_upscaler->get_param_tensors(tensors);
+    } else {
+        esrgan_upscaler = std::make_shared<ESRGAN>(backend_for(SDBackendModule::UPSCALER),
+                                                   tensor_storage_map,
+                                                   model_manager);
+        if (esrgan_upscaler->rrdb_net == nullptr) {
+            LOG_ERROR("init esrgan model from metadata failed: '%s'", esrgan_path.c_str());
+            return false;
+        }
+        esrgan_upscaler->get_param_tensors(tensors);
     }
-    esrgan_upscaler->set_max_graph_vram_bytes(max_graph_vram_bytes);
+    LOG_INFO("Upscaler architecture: %s", runner()->get_desc().c_str());
+    runner()->set_max_graph_vram_bytes(max_graph_vram_bytes);
     if (direct) {
-        esrgan_upscaler->set_conv2d_direct_enabled(true);
+        runner()->set_conv2d_direct_enabled(true);
     }
 
-    std::map<std::string, ggml_tensor*> tensors;
-    esrgan_upscaler->get_param_tensors(tensors);
     if (!model_manager->set_loader(model_loader) ||
         !model_manager->register_param_tensors(ModelComponent::Upscaler,
                                                std::move(tensors),
@@ -103,7 +141,7 @@ bool UpscalerGGML::load_from_file(const std::string& esrgan_path,
                                                backend_for(SDBackendModule::UPSCALER),
                                                params_backend_for(SDBackendModule::UPSCALER)) ||
         !model_manager->validate_registered_tensors()) {
-        LOG_ERROR("register esrgan tensors with model manager failed");
+        LOG_ERROR("register upscaler tensors with model manager failed");
         return false;
     }
     return true;
@@ -111,21 +149,33 @@ bool UpscalerGGML::load_from_file(const std::string& esrgan_path,
 
 sd::Tensor<float> UpscalerGGML::upscale_tensor(const sd::Tensor<float>& input_tensor) {
     sd::ParallelScope tensor_scope(&tensor_executor);
+    if (input_tensor.empty() || input_tensor.dim() != 4 ||
+        (input_tensor.shape()[2] != 3 && input_tensor.shape()[2] != 4)) {
+        LOG_ERROR("upscaler expects a 4D RGB or RGBA image tensor");
+        return {};
+    }
+
+    const bool has_alpha = input_tensor.shape()[2] == 4;
+    sd::Tensor<float> rgb;
+    if (has_alpha) {
+        rgb = sd::ops::slice(input_tensor, 2, 0, 3);
+    }
+    const sd::Tensor<float>& model_input = has_alpha ? rgb : input_tensor;
     sd::Tensor<float> upscaled;
-    const int scale = esrgan_upscaler->config.scale;
+    const int scale = get_scale();
     if (tile_size <= 0 || (input_tensor.shape()[0] <= tile_size && input_tensor.shape()[1] <= tile_size)) {
-        upscaled = esrgan_upscaler->compute(n_threads, input_tensor);
+        upscaled = compute(model_input);
     } else {
         auto on_processing = [&](const sd::Tensor<float>& input_tile) -> sd::Tensor<float> {
-            auto output_tile = esrgan_upscaler->compute(n_threads, input_tile);
+            auto output_tile = compute(input_tile);
             if (output_tile.empty()) {
-                LOG_ERROR("esrgan compute failed while processing a tile");
+                LOG_ERROR("upscaler compute failed while processing a tile");
                 return {};
             }
             return output_tile;
         };
 
-        upscaled = process_tiles_2d(input_tensor,
+        upscaled = process_tiles_2d(model_input,
                                     static_cast<int>(input_tensor.shape()[0] * scale),
                                     static_cast<int>(input_tensor.shape()[1] * scale),
                                     scale,
@@ -136,11 +186,19 @@ sd::Tensor<float> UpscalerGGML::upscale_tensor(const sd::Tensor<float>& input_te
                                     false,
                                     on_processing);
     }
-    esrgan_upscaler->runner_end();
+    runner()->runner_end();
     if (upscaled.empty()) {
-        LOG_ERROR("esrgan compute failed");
+        LOG_ERROR("upscaler compute failed");
         return {};
     }
+    if (has_alpha) {
+        auto alpha       = sd::ops::slice(input_tensor, 2, 3, 4);
+        auto alpha_shape = alpha.shape();
+        alpha_shape[0]   = upscaled.shape()[0];
+        alpha_shape[1]   = upscaled.shape()[1];
+        alpha            = sd::ops::interpolate(alpha, alpha_shape, sd::ops::InterpolateMode::Bilinear);
+        upscaled         = sd::ops::concat(upscaled, alpha, 2);
+    }
     return upscaled;
 }
 
@@ -148,7 +206,7 @@ sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor
     sd::ParallelScope tensor_scope(&tensor_executor);
     // upscale_factor, unused for RealESRGAN_x4plus_anime_6B.pth
     sd_image_t upscaled_image = {0, 0, 0, nullptr};
-    const int scale           = esrgan_upscaler->config.scale;
+    const int scale           = get_scale();
     int output_width          = (int)input_image.width * scale;
     int output_height         = (int)input_image.height * scale;
     LOG_INFO("upscaling from (%i x %i) to (%i x %i)",
@@ -236,10 +294,38 @@ bool upscale(upscaler_ctx_t* upscaler_ctx,
 }
 
 int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) {
-    if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr || upscaler_ctx->upscaler->esrgan_upscaler == nullptr) {
+    if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) {
         return 1;
     }
-    return upscaler_ctx->upscaler->esrgan_upscaler->config.scale;
+    return upscaler_ctx->upscaler->get_scale();
+}
+
+int get_upscaler_model_scale(const char* model_path) {
+    if (model_path == nullptr || model_path[0] == '\0') {
+        return 0;
+    }
+    try {
+        ModelLoader loader;
+        if (!loader.init_from_file_and_convert_name(model_path, "", VERSION_ESRGAN)) {
+            return 0;
+        }
+        const auto& tensors = loader.get_tensor_storage_map();
+        if (tensors.count("gblocks.0.weight") > 0) {
+            MoSRConfig config = MoSRConfig::detect_from_weights(tensors);
+            return config.is_valid() ? config.scale : 0;
+        }
+        auto first = tensors.find("conv_first.weight");
+        auto last  = tensors.find("conv_last.weight");
+        if (first == tensors.end() || last == tensors.end() ||
+            tensors.count("body.0.rdb1.conv1.weight") == 0 ||
+            first->second.n_dims != 4 || last->second.n_dims != 4 ||
+            first->second.ne[2] != 3 || last->second.ne[3] != 3) {
+            return 0;
+        }
+        return ESRGANConfig::detect_from_weights(tensors).scale;
+    } catch (const std::exception&) {
+        return 0;
+    }
 }
 
 void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) {
diff --git a/src/upscaler.h b/src/upscaler.h
index c23e9d0f25..b93065fcc9 100644
--- a/src/upscaler.h
+++ b/src/upscaler.h
@@ -4,6 +4,7 @@
 #include "core/ggml_extend_backend.h"
 #include "core/tensor.hpp"
 #include "model/upscaler/esrgan.hpp"
+#include "model/upscaler/mosr.hpp"
 #include "model_manager.h"
 #include "stable-diffusion.h"
 
@@ -15,6 +16,7 @@ struct UpscalerGGML {
     std::shared_ptr<ModelManager> model_manager;
     ggml_type model_data_type = GGML_TYPE_F16;
     std::shared_ptr<ESRGAN> esrgan_upscaler;
+    std::shared_ptr<MoSR> mosr_upscaler;
     std::string esrgan_path;
     int n_threads;
     sd::ParallelExecutor tensor_executor;
@@ -34,6 +36,9 @@ struct UpscalerGGML {
     bool load_from_file(const std::string& esrgan_path,
                         int n_threads);
     void set_max_graph_vram_bytes(size_t max_vram_bytes);
+    GGMLRunner* runner();
+    int get_scale();
+    sd::Tensor<float> compute(const sd::Tensor<float>& input_tensor);
     sd::Tensor<float> upscale_tensor(const sd::Tensor<float>& input_tensor);
     sd_image_t upscale(sd_image_t input_image, uint32_t upscale_factor);
 };
diff --git a/start-server.bat b/start-server.bat
new file mode 100644
index 0000000000..60ea9e1803
--- /dev/null
+++ b/start-server.bat
@@ -0,0 +1,45 @@
+@echo off
+setlocal DisableDelayedExpansion
+
+set "SD_SERVER_PATH=%SD_SERVER_PATH%"
+set "MODEL_DIR=./models"
+set "COMMANDLINE_ARGS="
+
+if exist "%~dp0start-server.local.bat" (
+    call "%~dp0start-server.local.bat"
+    if errorlevel 1 exit /b 1
+)
+
+if defined SD_SERVER_PATH goto explicit
+
+for %%F in (
+    "%~dp0sd-server.exe"
+    "%~dp0bin\Release\sd-server.exe"
+    "%~dp0bin\sd-server.exe"
+    "%~dp0build\bin\Release\sd-server.exe"
+    "%~dp0build\bin\sd-server.exe"
+) do (
+    if exist "%%~F" (
+        set "SD_SERVER_PATH=%%~F"
+        goto run
+    )
+)
+
+>&2 echo sd-server.exe was not found. Build it first or set SD_SERVER_PATH to its executable path.
+exit /b 1
+
+:explicit
+if exist "%SD_SERVER_PATH%" goto run
+>&2 echo SD_SERVER_PATH does not exist: "%SD_SERVER_PATH%"
+exit /b 1
+
+:run
+if "%~1"=="--version" goto version
+echo Running: "%SD_SERVER_PATH%" --model-dir "%MODEL_DIR%" %COMMANDLINE_ARGS% %*
+"%SD_SERVER_PATH%" --model-dir "%MODEL_DIR%" %COMMANDLINE_ARGS% %*
+exit /b %errorlevel%
+
+:version
+echo Running: "%SD_SERVER_PATH%" %*
+"%SD_SERVER_PATH%" %*
+exit /b %errorlevel%
diff --git a/start-server.sh b/start-server.sh
new file mode 100644
index 0000000000..b73dc27452
--- /dev/null
+++ b/start-server.sh
@@ -0,0 +1,53 @@
+#!/bin/sh
+set -eu
+
+SD_SERVER_PATH="${SD_SERVER_PATH:-}"
+MODEL_DIR="./models"
+COMMANDLINE_ARGS=""
+
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+if [ -f "$script_dir/start-server.local.sh" ]; then
+    . "$script_dir/start-server.local.sh"
+fi
+
+if [ "${1:-}" != "--version" ]; then
+    # Split configured arguments on whitespace without expanding wildcards.
+    set -f
+    set -- --model-dir "$MODEL_DIR" $COMMANDLINE_ARGS "$@"
+fi
+
+run_server() {
+    printf 'Running:'
+    for arg do
+        printf " '%s'" "$(printf '%s' "$arg" | sed "s/'/'\\\\''/g")"
+    done
+    printf '\n'
+    exec "$@"
+}
+
+if [ -n "$SD_SERVER_PATH" ]; then
+    case "$SD_SERVER_PATH" in
+        */*) ;;
+        *) SD_SERVER_PATH="./$SD_SERVER_PATH" ;;
+    esac
+    if [ ! -f "$SD_SERVER_PATH" ] || [ ! -x "$SD_SERVER_PATH" ]; then
+        printf 'SD_SERVER_PATH is not an executable file: %s\n' "$SD_SERVER_PATH" >&2
+        exit 1
+    fi
+    run_server "$SD_SERVER_PATH" "$@"
+fi
+
+for server in \
+    "$script_dir/sd-server" \
+    "$script_dir/bin/Release/sd-server" \
+    "$script_dir/bin/sd-server" \
+    "$script_dir/build/bin/Release/sd-server" \
+    "$script_dir/build/bin/sd-server"
+do
+    if [ -f "$server" ] && [ -x "$server" ]; then
+        run_server "$server" "$@"
+    fi
+done
+
+printf '%s\n' 'sd-server was not found. Build it first or set SD_SERVER_PATH to its executable path.' >&2
+exit 1