diff --git a/CLAUDE.md b/CLAUDE.md index 50d36d3..00babf5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -371,6 +371,21 @@ phase-onloaded): 1024x1024 in ~13 s (peak 12.5 GB alloc) after the allocator gc-gate fix (was ~48 s); pipeline load 31 s. Cast census is exactly 104 weights. +Image + prompt (same weights, one reference image): + +```r +pipe <- flux2_load_pipeline(encoder = TRUE) # adds the VAE encoder +img2img_flux2("Turn the scene into a snowy winter night", + image = "photo.png", pipeline = pipe, seed = 7) +``` + +The reference is VAE-encoded once and its tokens ride after the noisy +latents in every forward (position ids T = 10), so attention cost grows +with the reference size; the output defaults to the reference's size. +Verified against diffusers on the real weights (fp32 CPU, 256x256: PSNR +123 dB; encoder rel. error 2e-6). GPU memory at 1024x1024 not yet +measured. + Perf lesson that cost an afternoon: generation was 93.8% R garbage collection until the tokenizer stopped holding 151k-binding environments — R gc cost scales with live object count (12 vs 203 ms), diff --git a/DESCRIPTION b/DESCRIPTION index aa5c3d3..221bc0c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: diffuseR Title: Functional Interface to Diffusion Models in R -Version: 0.2.2.14 +Version: 0.2.2.15 Authors@R: c( person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"), comment = c(ORCID = "0009-0005-4248-604X")), diff --git a/NAMESPACE b/NAMESPACE index 439c73e..2736e3c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -53,6 +53,7 @@ export(flux_unpack_latents) export(flux2_bn_normalize) export(flux2_double_block) export(flux2_empirical_mu) +export(flux2_encode_image) export(flux2_feed_forward) export(flux2_is_quant_key) export(flux2_load_pipeline) @@ -60,13 +61,16 @@ export(flux2_modulation) export(flux2_pack_latents) export(flux2_parallel_self_attention) export(flux2_patchify_latents) +export(flux2_prepare_image_ids) export(flux2_prepare_latent_ids) export(flux2_prepare_text_ids) +export(flux2_preprocess_image) export(flux2_single_block) export(flux2_transformer) export(flux2_unpack_latents_with_ids) export(flux2_unpatchify_latents) export(flux2_vae_decoder) +export(flux2_vae_encoder) export(gemma3_config_ltx2) export(gemma3_encode_batch) export(gemma3_quantize_nf4) @@ -76,11 +80,13 @@ export(gemma4_quantize_nf4) export(gemma4_text_model) export(gemma4_tokenizer) export(img2img) +export(img2img_flux2) export(is_blackwell_gpu) export(latents_to_video) export(load_decoder_safetensors) export(load_decoder_weights) export(load_flux2_vae_decoder) +export(load_flux2_vae_encoder) export(load_gemma3_nf4) export(load_gemma3_text_encoder) export(load_gemma4_nf4) diff --git a/NEWS.md b/NEWS.md index f1bca17..f27aed1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,18 @@ +# diffuseR 0.2.2.15 + +* **FLUX.2 klein image + prompt editing.** `img2img_flux2(prompt, image)` + edits one reference image with the same klein-4B weights: the + reference is VAE-encoded once and conditions every step, following the + diffusers klein pipeline's `image =` path. + - `flux2_load_pipeline(encoder = TRUE)` loads the new VAE encoder + (`flux2_vae_encoder()`, `load_flux2_vae_encoder()`); the default + keeps text-to-image memory unchanged. + - `flux2_preprocess_image()` validates and prepares the reference + (64 px minimum side, 8:1 aspect limit, Lanczos resize to 1024^2 in + area, center crop to multiples of 16). + - `flux2_encode_image()` and `flux2_prepare_image_ids()` build the + reference tokens and their position ids. + # diffuseR 0.2.2.14 * **LTX-2.5 (distilled) text-to-video.** The 2.3 pipeline runs the 2.5 diff --git a/R/img2img_flux2.R b/R/img2img_flux2.R new file mode 100644 index 0000000..e57d798 --- /dev/null +++ b/R/img2img_flux2.R @@ -0,0 +1,161 @@ +#' Edit an image with FLUX.2 klein +#' +#' Image + prompt to image with the same klein-4B weights as +#' \code{\link{txt2img_flux2}}: the reference image is VAE-encoded once, +#' its tokens ride after the noisy latents in every transformer forward +#' (position ids at T = 10, see \code{\link{flux2_prepare_image_ids}}), +#' and only the latents' predictions drive the scheduler. Reference: +#' diffusers Flux2KleinPipeline with \code{image =} (Apache-2.0). +#' +#' The image is checked (both sides at least 64 px, aspect ratio at most +#' 8), resized with Lanczos to 1024 x 1024 in area if it is larger, and +#' center-cropped to multiples of 16. One reference image is supported. +#' +#' @param prompt Character. The edit instruction or prompt. +#' @param image Path to a PNG/JPEG, or an [H, W, 3] array in [0, 1]. +#' @param pipeline A \code{flux2_pipeline} loaded with +#' \code{encoder = TRUE}; NULL loads one (passing \code{...} through). +#' @param width,height Integers divisible by 16, or NULL (default) for +#' the processed reference image's size. +#' @param num_inference_steps Integer. Denoising steps (klein-4B: 4). +#' @param max_sequence_length Integer. Qwen3 token length (512). +#' @param seed Integer or NULL. See \code{\link{txt2img_flux2}}. +#' @param prompt_embeds Optional precomputed [B, S, 7680] embeddings. +#' @param save_file Logical. Write a PNG. +#' @param filename Output path (default derived from the prompt). +#' @param verbose Logical, or one of "silent", "progress", "steps". +#' @param ... Passed to \code{\link{flux2_load_pipeline}} when +#' \code{pipeline} is NULL. +#' +#' @return Invisibly, \code{list(image, metadata)} where \code{image} is +#' an [H, W, 3] array in [0, 1]. +#' +#' @export +img2img_flux2 <- function(prompt, image, pipeline = NULL, width = NULL, + height = NULL, num_inference_steps = 4L, + max_sequence_length = 512L, seed = NULL, + prompt_embeds = NULL, save_file = TRUE, + filename = NULL, verbose = TRUE, ...) { + level <- .verbosity(verbose) + reference <- flux2_preprocess_image(image) + width <- as.integer(width %||% reference$shape[4]) + height <- as.integer(height %||% reference$shape[3]) + if (level != "silent") { + message(sprintf("FLUX.2 klein edit: %dx%d from a %dx%d reference, %d steps", + width, height, as.integer(reference$shape[4]), + as.integer(reference$shape[3]), + as.integer(num_inference_steps))) + } + if (is.null(pipeline)) { + pipeline <- flux2_load_pipeline(..., encoder = TRUE, + verbose = level == "steps") + } + .flux2_generate(prompt, pipeline, width, height, num_inference_steps, + max_sequence_length, seed, prompt_embeds, save_file, + filename, level, reference = reference) +} + +#' Preprocess a FLUX.2 reference image +#' +#' Mirrors the diffusers klein pipeline: validate (both sides at least +#' 64 px, aspect ratio at most 8), Lanczos-resize to \code{max_area} if +#' larger (PIL's antialiased Lanczos-3, computed in floating point), floor +#' both sides to a multiple of 16, center-crop, and scale to [-1, 1]. +#' +#' @param image Path to a PNG/JPEG, or an [H, W, C] array in [0, 1]. +#' @param max_area Numeric. Pixel area cap. +#' +#' @return Float32 tensor [1, 3, H, W] in [-1, 1]. +#' +#' @export +flux2_preprocess_image <- function(image, max_area = 1024 * 1024) { + if (is.character(image)) { + if (!file.exists(image)) { + stop("image file not found: ", image, call. = FALSE) + } + image <- if (grepl("\\.png$", image, ignore.case = TRUE)) { + png::readPNG(image) + } else if (grepl("\\.jpe?g$", image, ignore.case = TRUE)) { + jpeg::readJPEG(image) + } else { + stop("image must be a .png, .jpg or .jpeg file", call. = FALSE) + } + } + if (!is.array(image) && !is.matrix(image)) { + stop("image must be a file path or an [H, W, 3] array", call. = FALSE) + } + if (length(dim(image)) == 2L) { + image <- array(rep(image, 3L), dim = c(dim(image), 3L)) + } + if (length(dim(image)) != 3L) { + stop("image must be an [H, W, 3] array", call. = FALSE) + } + if (dim(image)[3] == 4L) { + image <- image[,, 1:3, drop = FALSE] + } + if (dim(image)[3] != 3L) { + stop("image must have 3 colour channels", call. = FALSE) + } + + h <- dim(image)[1] + w <- dim(image)[2] + if (h < 64L || w < 64L) { + stop(sprintf("image too small: %dx%d; both sides must be at least 64 px", + w, h), call. = FALSE) + } + if (max(w / h, h / w) > 8) { + stop(sprintf("aspect ratio too extreme: %dx%d; at most 8:1", w, h), + call. = FALSE) + } + + x <- torch::torch_tensor(image, dtype = torch::torch_float32()) + x <- x$permute(c(3L, 1L, 2L)) # [3, H, W] + if (w * h > max_area) { + scale <- sqrt(max_area / (w * h)) + new_w <- as.integer(floor(w * scale)) + new_h <- as.integer(floor(h * scale)) + x <- .lanczos_resize(x, new_h, new_w) + h <- new_h + w <- new_w + } + crop_h <- (h %/% 16L) * 16L + crop_w <- (w %/% 16L) * 16L + top <- (h - crop_h) %/% 2L + left <- (w - crop_w) %/% 2L + x <- x$narrow(2L, top + 1L, crop_h)$narrow(3L, left + 1L, crop_w) + x$unsqueeze(1L)$mul(2)$sub(1) +} + +# PIL's Lanczos-3 resampling weights for one axis (ImagingResample +# precompute_coeffs): the kernel support widens by the downscale factor, +# which is what antialiases, and each row is normalized to sum to 1. +# Returns an [out_size, in_size] matrix. +.lanczos_weights <- function(in_size, out_size) { + scale <- in_size / out_size + filterscale <- max(scale, 1) + support <- 3 * filterscale + sinc <- function(v) ifelse(v == 0, 1, sin(pi * v) / (pi * v)) + lanczos <- function(v) ifelse(abs(v) < 3, sinc(v) * sinc(v / 3), 0) + m <- matrix(0, out_size, in_size) + for (i in seq_len(out_size)) { + center <- (i - 0.5) * scale + xmin <- max(as.integer(center - support + 0.5), 0L) + xmax <- min(as.integer(center + support + 0.5), in_size) + if (xmax <= xmin) { + next + } + src <- xmin:(xmax - 1L) + k <- lanczos((src - center + 0.5) / filterscale) + m[i, src + 1L] <- k / sum(k) + } + m +} + +# Separable Lanczos resize of a [C, H, W] float tensor. +.lanczos_resize <- function(x, new_h, new_w) { + f32 <- torch::torch_float32() + wh <- torch::torch_tensor(.lanczos_weights(x$shape[2], new_h), dtype = f32) + ww <- torch::torch_tensor(.lanczos_weights(x$shape[3], new_w), dtype = f32) + x <- torch::torch_matmul(x, ww$t()) # [C, H, new_w] + torch::torch_matmul(wh, x)$clamp(0, 1) # [C, new_h, new_w] +} diff --git a/R/rope_flux2.R b/R/rope_flux2.R index db13f15..68e6f5f 100644 --- a/R/rope_flux2.R +++ b/R/rope_flux2.R @@ -56,6 +56,30 @@ flux2_prepare_latent_ids <- function(height, width, device = "cpu") { ids$reshape(c(height * width, 4L)) } +#' FLUX.2 position ids for reference images +#' +#' (T, H, W, L) coordinates for reference-image tokens, concatenated in +#' order: the i-th reference (1-based) takes T = 10 * i, which keeps it +#' apart from the generated image's T = 0. Reference: +#' Flux2KleinPipeline._prepare_image_ids (scale 10). +#' +#' @param sizes List of \code{c(height, width)} packed grid sizes, one +#' per reference image. +#' @param device Device for the resulting tensor. +#' +#' @return Float tensor [sum(height * width), 4]. +#' +#' @export +flux2_prepare_image_ids <- function(sizes, device = "cpu") { + ids <- lapply(seq_along(sizes), function(i) { + x <- flux2_prepare_latent_ids(sizes[[i]][1], sizes[[i]][2], + device = device) + x[, 1] <- x[, 1]$add(10 * i) + x + }) + torch::torch_cat(ids, dim = 1L) +} + #' Empirical timestep shift for FLUX.2 #' #' BFL's piecewise-linear fit of the dynamic-shifting mu as a function of diff --git a/R/txt2img_flux2.R b/R/txt2img_flux2.R index b3ba0c6..f7306a9 100644 --- a/R/txt2img_flux2.R +++ b/R/txt2img_flux2.R @@ -86,6 +86,9 @@ NULL #' \code{snapshots//}, so the load works against a cache holding #' only that snapshot -- no \code{refs/} entry and no network. The #' transformer comes from \code{model_dir} and is unaffected. +#' @param encoder Logical. Also load the VAE encoder, which +#' \code{\link{img2img_flux2}} needs (FALSE keeps a text-to-image +#' pipeline's memory unchanged). #' #' @return A \code{flux2_pipeline} list. #' @@ -94,7 +97,8 @@ flux2_load_pipeline <- function(model_dir = NULL, device = "cuda", precision = c("auto", "fp8", "nf4", "bf16"), text_device = NULL, attn_chunk = NULL, phase_offload = TRUE, pin = NULL, - verbose = TRUE, revision = NULL) { + verbose = TRUE, revision = NULL, + encoder = FALSE) { ## Checked before any device work, where a configuration mistake is ## cheapest to report. .flux2_rev(revision) @@ -181,6 +185,21 @@ flux2_load_pipeline <- function(model_dir = NULL, device = "cuda", # the decode reads the decoder's own dtype and follows. pipe$decoder$to(dtype = torch::torch_bfloat16()) } + if (encoder) { + if (verbose) { + message("Loading FLUX.2 VAE encoder...") + } + pipe$encoder <- load_flux2_vae_encoder( + .flux2_cached("vae/diffusion_pytorch_model.safetensors", revision), + latent_channels = as.integer(vae_config$latent_channels %||% 32L), + verbose = verbose + ) + pipe$encoder$to(device = component_device) + # Same dtype policy as the decoder above + if (device == "cuda" && phase_offload) { + pipe$encoder$to(dtype = torch::torch_bfloat16()) + } + } if (verbose) { message("Loading Qwen3 text encoder...") @@ -195,6 +214,9 @@ flux2_load_pipeline <- function(model_dir = NULL, device = "cuda", .flux2_cached("tokenizer/tokenizer.json", revision)) components <- c("transformer", "decoder") + if (encoder) { + components <- c(components, "encoder") + } if (!identical(text_device, "cpu")) { components <- c(components, "text_encoder") } @@ -204,14 +226,18 @@ flux2_load_pipeline <- function(model_dir = NULL, device = "cuda", structure(pipe, class = "flux2_pipeline") } -# Flow-matching Euler loop; klein is CFG-free (one forward per step) +# Flow-matching Euler loop; klein is CFG-free (one forward per step). +# Reference-image tokens (img2img) ride after the noisy latents in every +# forward and only the latents' predictions are kept. .flux2_denoise <- function(transformer, latents, schedule, prompt_embeds, image_rotary_emb, compute_dtype, - chunk_size = NULL, verbose = TRUE) { + chunk_size = NULL, verbose = TRUE, + ref_tokens = NULL) { timesteps <- as.numeric(schedule$timesteps$cpu()) n <- length(timesteps) pb <- .denoise_progress(n, NULL, verbose) f32 <- torch::torch_float32() + seq_img <- latents$shape[2] torch::with_no_grad({ for (i in seq_len(n)) { @@ -219,13 +245,21 @@ flux2_load_pipeline <- function(model_dir = NULL, device = "cuda", t_model <- torch::torch_tensor(t / 1000, dtype = compute_dtype, device = latents$device)$reshape(1L) + hidden <- latents$to(dtype = compute_dtype) + if (!is.null(ref_tokens)) { + hidden <- torch::torch_cat(list(hidden, ref_tokens), dim = 2L) + } noise_pred <- transformer( - hidden_states = latents$to(dtype = compute_dtype), + hidden_states = hidden, encoder_hidden_states = prompt_embeds, timestep = t_model, image_rotary_emb = image_rotary_emb, chunk_size = chunk_size ) + rm(hidden) + if (!is.null(ref_tokens)) { + noise_pred <- noise_pred$narrow(2L, 1L, seq_img) + } step <- flowmatch_scheduler_step( noise_pred$to(dtype = f32), t, latents, schedule @@ -277,14 +311,26 @@ txt2img_flux2 <- function(prompt, pipeline = NULL, width = 1024L, prompt_embeds = NULL, save_file = TRUE, filename = NULL, verbose = TRUE, ...) { level <- .verbosity(verbose) - verbose <- level == "steps" if (level != "silent") { message(sprintf("FLUX.2 klein: %dx%d, %d steps", as.integer(width), as.integer(height), as.integer(num_inference_steps))) } if (is.null(pipeline)) { - pipeline <- flux2_load_pipeline(..., verbose = verbose) + pipeline <- flux2_load_pipeline(..., verbose = level == "steps") } + .flux2_generate(prompt, pipeline, width, height, num_inference_steps, + max_sequence_length, seed, prompt_embeds, save_file, + filename, level) +} + +# The shared FLUX.2 generation body. `reference` is NULL (txt2img) or a +# preprocessed [1, 3, H, W] image in [-1, 1] (img2img), encoded to +# reference tokens with the pipeline's VAE encoder. +.flux2_generate <- function(prompt, pipeline, width, height, + num_inference_steps, max_sequence_length, seed, + prompt_embeds, save_file, filename, level, + reference = NULL) { + verbose <- level == "steps" device <- pipeline$device width <- as.integer(width) height <- as.integer(height) @@ -367,7 +413,28 @@ txt2img_flux2 <- function(prompt, pipeline = NULL, width = 1024L, txt_ids <- flux2_prepare_text_ids(txt_len) latent_ids <- flux2_prepare_latent_ids(h2, w2) - ids <- torch::torch_cat(list(txt_ids, latent_ids), dim = 1L) + ids <- list(txt_ids, latent_ids) + + # --- Reference image (img2img): encode once, reuse every step ------------------- + ref_tokens <- NULL + if (!is.null(reference)) { + if (is.null(pipeline$encoder)) { + stop("this pipeline has no VAE encoder; load it with ", + "flux2_load_pipeline(encoder = TRUE)", call. = FALSE) + } + if (verbose) { + message("Encoding reference image...") + } + encoder <- onload("encoder") + ref <- flux2_encode_image(encoder, reference, + bn_eps = pipeline$vae_bn_eps %||% 1e-4) + offload("encoder") + ref_tokens <- ref$tokens$to(device = device, dtype = compute_dtype) + ids <- c(ids, list(flux2_prepare_image_ids(list(c(ref$height, + ref$width))))) + rm(ref) + } + ids <- torch::torch_cat(ids, dim = 1L) rope <- flux_pos_embed( ids, axes_dim = pipeline$transformer$axes_dims_rope %||% c(32L, 32L, 32L, 32L), @@ -403,8 +470,9 @@ txt2img_flux2 <- function(prompt, pipeline = NULL, width = 1024L, latents <- .flux2_denoise( transformer, latents, sched, prompt_embeds, rope, compute_dtype, chunk_size = pipeline$attn_chunk, - verbose = level + verbose = level, ref_tokens = ref_tokens ) + rm(ref_tokens) if (fp8_manual && phase_offload) { .flux_fp8_to_device(pipeline$transformer, "cpu") } diff --git a/R/vae_flux2_encoder.R b/R/vae_flux2_encoder.R new file mode 100644 index 0000000..a4eb2e4 --- /dev/null +++ b/R/vae_flux2_encoder.R @@ -0,0 +1,217 @@ +#' VAE Down Block +#' +#' diffusers DownEncoderBlock2D: resnets, then an optional stride-2 +#' downsample conv. The downsample pads (0, 1, 0, 1) and convolves with +#' padding 0 (Downsample2D with \code{downsample_padding = 0}). +#' +#' @param in_channels Input channels +#' @param out_channels Output channels +#' @param num_resnets Number of resnet blocks +#' @param add_downsample Whether to add the downsampler +#' @param norm_groups Group norm groups +#' @keywords internal +VAEDownBlock <- torch::nn_module( + "VAEDownBlock", + + initialize = function(in_channels, out_channels, num_resnets = 2, + add_downsample = TRUE, norm_groups = 32) { + self$resnets <- torch::nn_module_list() + for (i in seq_len(num_resnets)) { + res_in <- if (i == 1) in_channels else out_channels + self$resnets$append(VAEResnetBlock(res_in, out_channels, norm_groups)) + } + if (add_downsample) { + self$downsamplers <- torch::nn_module_list(list( + torch::nn_module( + "Downsampler", + initialize = function(channels) { + self$conv <- torch::nn_conv2d(channels, channels, + kernel_size = 3, stride = 2, padding = 0) + }, + forward = function(x) { + self$conv(torch::nnf_pad(x, c(0, 1, 0, 1))) + } + )(out_channels) + )) + } +}, + + forward = function(x) { + for (i in seq_along(self$resnets)) { + x <- self$resnets[[i]](x) + } + if (!is.null(self$downsamplers)) { + x <- self$downsamplers[[1]](x) + } + x +} +) + +#' FLUX.2 VAE encoder +#' +#' The AutoencoderKLFlux2 encode path: the standard diffusers AutoencoderKL +#' encoder body (conv_in, down blocks, attention mid block, GroupNorm + +#' SiLU, conv_out to 2 x latent channels) followed by quant_conv, plus the +#' BatchNorm running statistics used to normalize the patchified latent. +#' The forward returns the posterior mean (the reference pipeline encodes +#' with \code{sample_mode = "argmax"}). Reference: +#' src/diffusers/models/autoencoders/vae.py (Encoder) and +#' autoencoder_kl_flux2.py (Apache-2.0). +#' +#' @param latent_channels Integer (32 for FLUX.2). +#' @param block_channels Encoder block_out_channels. +#' @param layers_per_block Integer. Resnets per down block. +#' @param norm_groups Integer. Group norm groups. +#' +#' @return Module whose forward(x) encodes [B, 3, H, W] images in +#' [-1, 1] to [B, 32, H / 8, W / 8] latent means. +#' +#' @export +flux2_vae_encoder <- torch::nn_module( + "flux2_vae_encoder", + initialize = function(latent_channels = 32L, + block_channels = c(128L, 256L, 512L, 512L), + layers_per_block = 2L, norm_groups = 32L) { + body <- torch::nn_module( + "flux2_vae_encoder_body", + initialize = function() { + n <- length(block_channels) + self$conv_in <- torch::nn_conv2d(3L, block_channels[1], + kernel_size = 3, padding = 1) + self$down_blocks <- torch::nn_module_list() + for (i in seq_len(n)) { + in_ch <- block_channels[max(i - 1L, 1L)] + self$down_blocks$append(VAEDownBlock(in_ch, block_channels[i], + num_resnets = layers_per_block, + add_downsample = i < n, norm_groups = norm_groups)) + } + self$mid_block <- VAEMidBlock(block_channels[n], norm_groups) + self$conv_norm_out <- torch::nn_group_norm(norm_groups, + block_channels[n], eps = 1e-6) + self$conv_out <- torch::nn_conv2d(block_channels[n], + 2L * latent_channels, kernel_size = 3, padding = 1) + }, + forward = function(x) { + x <- self$conv_in(x) + for (i in seq_along(self$down_blocks)) { + x <- self$down_blocks[[i]](x) + } + x <- self$mid_block(x) + x <- torch::nnf_silu(self$conv_norm_out(x)) + self$conv_out(x) + } + ) + self$encoder <- body() + self$quant_conv <- torch::nn_conv2d(2L * latent_channels, + 2L * latent_channels, kernel_size = 1) + bn_stats <- torch::nn_module( + "flux2_bn_stats", + initialize = function(n) { + self$running_mean <- torch::nn_buffer(torch::torch_zeros(n)) + self$running_var <- torch::nn_buffer(torch::torch_ones(n)) + }, + forward = function(x) x + ) + self$bn <- bn_stats(latent_channels * 4L) + self$latent_channels <- latent_channels +}, + forward = function(x) { + moments <- self$quant_conv(self$encoder(x)) + # DiagonalGaussianDistribution: mean is the first half of the channels + moments$narrow(2L, 1L, self$latent_channels) +} +) + +#' Load the FLUX.2 VAE encoder from safetensors +#' +#' Loads the encoder half plus quant_conv and the BatchNorm running +#' statistics; decoder and post_quant_conv keys are skipped. +#' +#' @param path Path to the VAE .safetensors file (or a directory +#' containing diffusion_pytorch_model.safetensors). +#' @param latent_channels,block_channels,layers_per_block,norm_groups +#' Constructor arguments for \code{\link{flux2_vae_encoder}}. +#' @param verbose Logical. +#' +#' @return The loaded \code{flux2_vae_encoder} in eval mode. +#' +#' @export +load_flux2_vae_encoder <- function(path, latent_channels = 32L, + block_channels = c(128L, 256L, 512L, 512L), + layers_per_block = 2L, norm_groups = 32L, + verbose = TRUE) { + path <- path.expand(path) + if (dir.exists(path)) { + path <- file.path(path, "diffusion_pytorch_model.safetensors") + } + enc <- flux2_vae_encoder(latent_channels = latent_channels, + block_channels = block_channels, + layers_per_block = layers_per_block, + norm_groups = norm_groups) + + handle <- safetensors::safetensors$new(path, framework = "torch") + keys <- setdiff(handle$keys(), "__metadata__") + keep <- keys[startsWith(keys, "encoder.") | + startsWith(keys, "quant_conv.") | + keys %in% c("bn.running_mean", "bn.running_var")] + + dests <- c(enc$named_parameters(), enc$named_buffers()) + filled <- character(0) + unmapped <- character(0) + torch::with_no_grad({ + for (key in keep) { + dest <- dests[[key]] + if (is.null(dest)) { + unmapped <- c(unmapped, key) + next + } + dest$copy_(handle$get_tensor(key)) + filled <- c(filled, key) + } + }) + unfilled <- setdiff(names(dests), filled) + if (length(unmapped)) { + stop("FLUX.2 VAE encoder load: ", length(unmapped), + " unmapped keys, e.g. ", + paste(utils::head(unmapped, 3), collapse = ", ")) + } + if (length(unfilled)) { + stop("FLUX.2 VAE encoder load: ", length(unfilled), + " unfilled params, e.g. ", + paste(utils::head(unfilled, 3), collapse = ", ")) + } + if (verbose) { + message("Loaded ", length(filled), " FLUX.2 VAE encoder tensors from ", + path) + } + enc$eval() + enc +} + +#' Encode a reference image into FLUX.2 tokens +#' +#' VAE-encode (posterior mean), patchify 2x2, normalize with the VAE +#' BatchNorm statistics and pack to channels-last tokens. Reference: +#' Flux2KleinPipeline._encode_vae_image and prepare_image_latents. +#' +#' @param encoder A loaded \code{\link{flux2_vae_encoder}}. +#' @param image Tensor [1, 3, H, W] in [-1, 1]; H and W multiples of 16. +#' @param bn_eps Numeric. BatchNorm epsilon. +#' +#' @return \code{list(tokens, height, width)}: tokens [1, H/16 * W/16, 128] +#' on the encoder's device and dtype, and the packed grid size. +#' +#' @export +flux2_encode_image <- function(encoder, image, bn_eps = 1e-4) { + param <- encoder$encoder$conv_in$weight + torch::with_no_grad({ + latents <- encoder(image$to(device = param$device, dtype = param$dtype)) + latents <- flux2_patchify_latents(latents) + latents <- flux2_bn_normalize(latents, encoder$bn$running_mean, + encoder$bn$running_var, eps = bn_eps) + h <- latents$shape[3] + w <- latents$shape[4] + tokens <- flux2_pack_latents(latents) + }) + list(tokens = tokens, height = h, width = w) +} diff --git a/inst/REFERENCES.md b/inst/REFERENCES.md index 066630f..b316ee0 100644 --- a/inst/REFERENCES.md +++ b/inst/REFERENCES.md @@ -40,6 +40,10 @@ module; this file documents the actual lineage, idea by idea. | MMDiT transformer (shared modulation, SwiGLU feed-forwards, ViT-22B parallel single blocks, 4-axis RoPE) | Ported from HuggingFace **diffusers** (Apache-2.0): `models/transformers/transformer_flux2.py` | | Klein pipeline flow: Qwen3 prompt encoding contract (chat template, mid-stack hidden states 9/18/27), 4-channel position ids, latent patchify/pack chain, empirical dynamic-shift mu, BatchNorm latent statistics | diffusers `pipelines/flux2/pipeline_flux2_klein.py` (Apache-2.0); the empirical mu formula originates from BFL `sampling.py` (numeric facts) | | 32-channel `AutoencoderKLFlux2` decode path (post_quant_conv + standard decoder + BN running stats) | diffusers `models/autoencoders/autoencoder_kl_flux2.py` (Apache-2.0); decoder body shared with the SD/FLUX.1 port | +| `AutoencoderKLFlux2` encode path (Encoder with DownEncoderBlock2D / Downsample2D pad-then-stride-2 conv, quant_conv, posterior mean) | Ported from diffusers `models/autoencoders/vae.py`, `models/downsampling.py`, `autoencoder_kl_flux2.py` (Apache-2.0); resnet and mid blocks shared with the decoder port | +| Reference-image conditioning (`img2img_flux2`): encode, patchify, BN-normalize and pack the reference; position ids T = 10 * i; reference tokens concatenated after the latents each step and sliced off the prediction; mu from the target length only | diffusers `pipelines/flux2/pipeline_flux2_klein.py` (`image=`, `prepare_image_latents`, `_prepare_image_ids`) (Apache-2.0) | +| Reference-image preprocessing: 64 px minimum side, 8:1 aspect limit, 1024^2 area cap, floor to multiples of 16, center crop | diffusers `pipelines/flux2/image_processor.py` (Apache-2.0) | +| Antialiased Lanczos-3 resize weights (support widened by the downscale factor, per-row normalization) | Pillow `Resample.c` `precompute_coeffs` algorithm (MIT-CMU / HPND), reimplemented in floating point | | Qwen3 encoder (GQA, per-head q/k RMS norms, SwiGLU, RoPE theta 1e6, causal + padding mask) | Ported from HuggingFace **transformers** (Apache-2.0): `models/qwen3/` | | Qwen2 byte-level BPE tokenization (GPT-2 byte table, GPT-4-style split regex, rank-based merges) | GPT-2 BPE (Radford et al. 2019; openai/gpt-2 `encoder.py`, MIT); format facts from HuggingFace tokenizers documentation | | Weights | black-forest-labs/FLUX.2-klein-4B (Apache-2.0, ungated; downloaded by the user, never redistributed) | diff --git a/inst/tinytest/fixtures/vae_flux2_enc_io.safetensors b/inst/tinytest/fixtures/vae_flux2_enc_io.safetensors new file mode 100644 index 0000000..24a1b98 Binary files /dev/null and b/inst/tinytest/fixtures/vae_flux2_enc_io.safetensors differ diff --git a/inst/tinytest/test_img2img_flux2.R b/inst/tinytest/test_img2img_flux2.R new file mode 100644 index 0000000..c30ecb2 --- /dev/null +++ b/inst/tinytest/test_img2img_flux2.R @@ -0,0 +1,130 @@ +# FLUX.2 klein image + prompt input: VAE encoder parity against a tiny +# random-init diffusers AutoencoderKLFlux2 (fixtures from +# tools/gen_fixtures_flux2_vae_encoder.py), reference ids, preprocessing, +# and the reference-token wiring of the denoise loop. + +if (!requireNamespace("torch", quietly = TRUE) || !torch::torch_is_installed()) { + exit_file("torch not fully installed") +} +if (!requireNamespace("safetensors", quietly = TRUE)) { + exit_file("safetensors not installed") +} + +library(diffuseR) + +fixture <- function(name) { + p <- system.file("tinytest", "fixtures", name, package = "diffuseR") + if (p == "") p <- file.path("fixtures", name) + p +} +vae_path <- fixture("vae_flux2_tiny.safetensors") +io_path <- fixture("vae_flux2_enc_io.safetensors") +if (!file.exists(io_path)) exit_file("flux2 vae encoder fixtures missing") +io <- safetensors::safe_load_file(io_path, framework = "torch") + +max_abs_diff <- function(a, b) { + as.numeric(torch::torch_max(torch::torch_abs(a - b))) +} + +# --- Encoder parity ----------------------------------------------------------- +enc <- load_flux2_vae_encoder(vae_path, block_channels = c(8L, 16L, 32L, 32L), + norm_groups = 8L, verbose = FALSE) +mean <- torch::with_no_grad(enc(io$image)) +expect_equal(as.integer(mean$shape), c(1L, 32L, 4L, 6L)) +expect_true(max_abs_diff(mean, io$mean) < 1e-5) + +ref <- flux2_encode_image(enc, io$image) +expect_equal(c(ref$height, ref$width), c(2L, 3L)) +expect_true(max_abs_diff(ref$tokens, io$tokens) < 1e-5) + +# Reference ids: T = 10 for the first image, then the row/column grid +ids <- flux2_prepare_image_ids(list(c(ref$height, ref$width))) +expect_equal(as.matrix(ids), as.matrix(io$image_ids)) +ids2 <- flux2_prepare_image_ids(list(c(1L, 2L), c(1L, 1L))) +expect_equal(as.numeric(ids2[, 1]), c(10, 10, 20)) + +# The loader refuses a file it cannot fill +expect_error(load_flux2_vae_encoder(vae_path, verbose = FALSE), "unmapped|unfilled|size") + +# --- Preprocessing -------------------------------------------------------------- +img <- array(0.75, c(70L, 100L, 3L)) +p <- flux2_preprocess_image(img) +expect_equal(as.integer(p$shape), c(1L, 3L, 64L, 96L)) +expect_true(max(abs(as.numeric(p) - 0.5)) < 1e-6) + +# Center crop: a column marker survives at the right offset +img <- array(0, c(64L, 72L, 3L)) +img[, 5L, ] <- 1 # crop drops 4 columns from the left +p <- flux2_preprocess_image(img) +expect_equal(as.integer(p$shape), c(1L, 3L, 64L, 64L)) +expect_equal(as.numeric(p[1, 1, 1, 1]), 1) + +# Grayscale and alpha inputs become RGB +expect_equal(as.integer(flux2_preprocess_image(matrix(0.5, 64L, 64L))$shape), + c(1L, 3L, 64L, 64L)) +expect_equal(as.integer(flux2_preprocess_image(array(0.5, c(64L, 64L, 4L)))$shape), + c(1L, 3L, 64L, 64L)) + +# Area cap: 300 x 200 over a 100^2-pixel cap -> 122 x 81 -> 112 x 80 +p <- flux2_preprocess_image(array(0.25, c(200L, 300L, 3L)), max_area = 100^2) +expect_equal(as.integer(p$shape), c(1L, 3L, 80L, 112L)) +expect_true(max(abs(as.numeric(p) + 0.5)) < 1e-5) # a constant stays constant + +expect_error(flux2_preprocess_image(array(0, c(40L, 100L, 3L))), "too small") +expect_error(flux2_preprocess_image(array(0, c(64L, 600L, 3L))), "aspect ratio") +expect_error(flux2_preprocess_image("no-such-file.png"), "not found") + +# Lanczos weights: rows sum to 1; same size is the identity +w <- diffuseR:::.lanczos_weights(10L, 4L) +expect_equal(rowSums(w), rep(1, 4)) +expect_equal(diffuseR:::.lanczos_weights(5L, 5L), diag(5), tolerance = 1e-12) + +# --- Denoise wiring: reference tokens ride after the latents --------------------- +seen <- new.env() +fake <- function(hidden_states, encoder_hidden_states, timestep, + image_rotary_emb, chunk_size = NULL) { + seen$len <- hidden_states$shape[2] + hidden_states$mul(0) +} +sched <- flowmatch_scheduler_create(use_dynamic_shifting = TRUE, + time_shift_type = "exponential") +sched <- flowmatch_set_timesteps(sched, 2L, mu = 1, sigmas = c(1, 0.5)) +lat <- torch::torch_randn(1L, 6L, 128L) +out <- diffuseR:::.flux2_denoise(fake, lat, sched, NULL, NULL, + torch::torch_float32(), verbose = FALSE, + ref_tokens = torch::torch_randn(1L, 4L, 128L)) +expect_equal(seen$len, 10L) +expect_equal(as.integer(out$shape), c(1L, 6L, 128L)) +expect_true(max_abs_diff(out, lat) < 1e-6) # zero velocity leaves latents unchanged + +# --- End to end with tiny components ------------------------------------------------ +torch::torch_manual_seed(8) +transformer <- flux2_transformer(in_channels = 128L, num_layers = 1L, + num_single_layers = 1L, attention_head_dim = 8L, + num_attention_heads = 2L, joint_attention_dim = 24L, + mlp_ratio = 3.0, axes_dims_rope = c(2L, 2L, 2L, 2L), + rope_theta = 2000) +transformer$eval() +decoder <- flux2_vae_decoder(latent_channels = 32L, + block_channels = c(32L, 32L, 16L, 8L), norm_groups = 8L) +decoder$eval() +pipeline <- structure(list(transformer = transformer, decoder = decoder, + device = "cpu", text_device = "cpu", phase_offload = FALSE, + fp8_resident = FALSE, format = "full", attn_chunk = NULL, + vae_bn_eps = 1e-4), + class = "flux2_pipeline") +emb <- torch::torch_randn(1L, 7L, 24L) + +# Without an encoder the edit says how to load one +expect_error(img2img_flux2("x", array(0.5, c(64L, 96L, 3L)), pipeline = pipeline, + num_inference_steps = 1L, prompt_embeds = emb, + save_file = FALSE, verbose = FALSE), + "encoder = TRUE") + +pipeline$encoder <- enc +res <- img2img_flux2("tiny edit", array(0.5, c(64L, 96L, 3L)), pipeline = pipeline, + num_inference_steps = 2L, seed = 42L, prompt_embeds = emb, + save_file = FALSE, verbose = FALSE) +expect_equal(dim(res$image), c(64L, 96L, 3L)) # output size follows the reference +expect_true(all(is.finite(res$image))) +expect_true(all(res$image >= 0) && all(res$image <= 1)) diff --git a/man/VAEDownBlock.Rd b/man/VAEDownBlock.Rd new file mode 100644 index 0000000..ae2f90a --- /dev/null +++ b/man/VAEDownBlock.Rd @@ -0,0 +1,30 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{VAEDownBlock} +\alias{VAEDownBlock} +\title{VAE Down Block} +\usage{ +VAEDownBlock( + in_channels, + out_channels, + num_resnets = 2, + add_downsample = TRUE, + norm_groups = 32 +) +} +\arguments{ +\item{in_channels}{Input channels} + +\item{out_channels}{Output channels} + +\item{num_resnets}{Number of resnet blocks} + +\item{add_downsample}{Whether to add the downsampler} + +\item{norm_groups}{Group norm groups} +} +\description{ +diffusers DownEncoderBlock2D: resnets, then an optional stride-2 +downsample conv. The downsample pads (0, 1, 0, 1) and convolves with +padding 0 (Downsample2D with \code{downsample_padding = 0}). +} +\keyword{internal} diff --git a/man/flux2_encode_image.Rd b/man/flux2_encode_image.Rd new file mode 100644 index 0000000..5b09776 --- /dev/null +++ b/man/flux2_encode_image.Rd @@ -0,0 +1,23 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{flux2_encode_image} +\alias{flux2_encode_image} +\title{Encode a reference image into FLUX.2 tokens} +\usage{ +flux2_encode_image(encoder, image, bn_eps = 1e-04) +} +\arguments{ +\item{encoder}{A loaded \code{\link{flux2_vae_encoder}}.} + +\item{image}{Tensor [1, 3, H, W] in [-1, 1]; H and W multiples of 16.} + +\item{bn_eps}{Numeric. BatchNorm epsilon.} +} +\value{ +\code{list(tokens, height, width)}: tokens [1, H/16 * W/16, 128] + on the encoder's device and dtype, and the packed grid size. +} +\description{ +VAE-encode (posterior mean), patchify 2x2, normalize with the VAE +BatchNorm statistics and pack to channels-last tokens. Reference: +Flux2KleinPipeline._encode_vae_image and prepare_image_latents. +} diff --git a/man/flux2_load_pipeline.Rd b/man/flux2_load_pipeline.Rd index 809e978..6accf19 100644 --- a/man/flux2_load_pipeline.Rd +++ b/man/flux2_load_pipeline.Rd @@ -12,7 +12,8 @@ flux2_load_pipeline( phase_offload = TRUE, pin = NULL, verbose = TRUE, - revision = NULL + revision = NULL, + encoder = FALSE ) } \arguments{ @@ -45,6 +46,10 @@ cache. With one they resolve straight out of \code{snapshots//}, so the load works against a cache holding only that snapshot -- no \code{refs/} entry and no network. The transformer comes from \code{model_dir} and is unaffected.} + +\item{encoder}{Logical. Also load the VAE encoder, which +\code{\link{img2img_flux2}} needs (FALSE keeps a text-to-image +pipeline's memory unchanged).} } \value{ A \code{flux2_pipeline} list. diff --git a/man/flux2_prepare_image_ids.Rd b/man/flux2_prepare_image_ids.Rd new file mode 100644 index 0000000..146a402 --- /dev/null +++ b/man/flux2_prepare_image_ids.Rd @@ -0,0 +1,22 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{flux2_prepare_image_ids} +\alias{flux2_prepare_image_ids} +\title{FLUX.2 position ids for reference images} +\usage{ +flux2_prepare_image_ids(sizes, device = "cpu") +} +\arguments{ +\item{sizes}{List of \code{c(height, width)} packed grid sizes, one +per reference image.} + +\item{device}{Device for the resulting tensor.} +} +\value{ +Float tensor [sum(height * width), 4]. +} +\description{ +(T, H, W, L) coordinates for reference-image tokens, concatenated in +order: the i-th reference (1-based) takes T = 10 * i, which keeps it +apart from the generated image's T = 0. Reference: +Flux2KleinPipeline._prepare_image_ids (scale 10). +} diff --git a/man/flux2_preprocess_image.Rd b/man/flux2_preprocess_image.Rd new file mode 100644 index 0000000..56e23ae --- /dev/null +++ b/man/flux2_preprocess_image.Rd @@ -0,0 +1,21 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{flux2_preprocess_image} +\alias{flux2_preprocess_image} +\title{Preprocess a FLUX.2 reference image} +\usage{ +flux2_preprocess_image(image, max_area = 1024 * 1024) +} +\arguments{ +\item{image}{Path to a PNG/JPEG, or an [H, W, C] array in [0, 1].} + +\item{max_area}{Numeric. Pixel area cap.} +} +\value{ +Float32 tensor [1, 3, H, W] in [-1, 1]. +} +\description{ +Mirrors the diffusers klein pipeline: validate (both sides at least +64 px, aspect ratio at most 8), Lanczos-resize to \code{max_area} if +larger (PIL's antialiased Lanczos-3, computed in floating point), floor +both sides to a multiple of 16, center-crop, and scale to [-1, 1]. +} diff --git a/man/flux2_vae_encoder.Rd b/man/flux2_vae_encoder.Rd new file mode 100644 index 0000000..8b956cc --- /dev/null +++ b/man/flux2_vae_encoder.Rd @@ -0,0 +1,35 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{flux2_vae_encoder} +\alias{flux2_vae_encoder} +\title{FLUX.2 VAE encoder} +\usage{ +flux2_vae_encoder( + latent_channels = 32L, + block_channels = c(128L, 256L, 512L, 512L), + layers_per_block = 2L, + norm_groups = 32L +) +} +\arguments{ +\item{latent_channels}{Integer (32 for FLUX.2).} + +\item{block_channels}{Encoder block_out_channels.} + +\item{layers_per_block}{Integer. Resnets per down block.} + +\item{norm_groups}{Integer. Group norm groups.} +} +\value{ +Module whose forward(x) encodes [B, 3, H, W] images in + [-1, 1] to [B, 32, H / 8, W / 8] latent means. +} +\description{ +The AutoencoderKLFlux2 encode path: the standard diffusers AutoencoderKL +encoder body (conv_in, down blocks, attention mid block, GroupNorm + +SiLU, conv_out to 2 x latent channels) followed by quant_conv, plus the +BatchNorm running statistics used to normalize the patchified latent. +The forward returns the posterior mean (the reference pipeline encodes +with \code{sample_mode = "argmax"}). Reference: +src/diffusers/models/autoencoders/vae.py (Encoder) and +autoencoder_kl_flux2.py (Apache-2.0). +} diff --git a/man/img2img_flux2.Rd b/man/img2img_flux2.Rd new file mode 100644 index 0000000..162aa7d --- /dev/null +++ b/man/img2img_flux2.Rd @@ -0,0 +1,67 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{img2img_flux2} +\alias{img2img_flux2} +\title{Edit an image with FLUX.2 klein} +\usage{ +img2img_flux2( + prompt, + image, + pipeline = NULL, + width = NULL, + height = NULL, + num_inference_steps = 4L, + max_sequence_length = 512L, + seed = NULL, + prompt_embeds = NULL, + save_file = TRUE, + filename = NULL, + verbose = TRUE, + ... +) +} +\arguments{ +\item{prompt}{Character. The edit instruction or prompt.} + +\item{image}{Path to a PNG/JPEG, or an [H, W, 3] array in [0, 1].} + +\item{pipeline}{A \code{flux2_pipeline} loaded with +\code{encoder = TRUE}; NULL loads one (passing \code{...} through).} + +\item{num_inference_steps}{Integer. Denoising steps (klein-4B: 4).} + +\item{max_sequence_length}{Integer. Qwen3 token length (512).} + +\item{seed}{Integer or NULL. See \code{\link{txt2img_flux2}}.} + +\item{prompt_embeds}{Optional precomputed [B, S, 7680] embeddings.} + +\item{save_file}{Logical. Write a PNG.} + +\item{filename}{Output path (default derived from the prompt).} + +\item{verbose}{Logical, or one of "silent", "progress", "steps".} + +\item{...}{Passed to \code{\link{flux2_load_pipeline}} when +\code{pipeline} is NULL.} + +\item{width,height}{Integers divisible by 16, or NULL (default) for +the processed reference image's size.} +} +\value{ +Invisibly, \code{list(image, metadata)} where \code{image} is + an [H, W, 3] array in [0, 1]. +} +\description{ +Image + prompt to image with the same klein-4B weights as +\code{\link{txt2img_flux2}}: the reference image is VAE-encoded once, +its tokens ride after the noisy latents in every transformer forward +(position ids at T = 10, see \code{\link{flux2_prepare_image_ids}}), +and only the latents' predictions drive the scheduler. Reference: +diffusers Flux2KleinPipeline with \code{image =} (Apache-2.0). +} +\details{ +The image is checked (both sides at least 64 px, aspect ratio at most +8), resized with Lanczos to 1024 x 1024 in area if it is larger, and +center-cropped to multiples of 16. One reference image is supported. + +} diff --git a/man/load_flux2_vae_encoder.Rd b/man/load_flux2_vae_encoder.Rd new file mode 100644 index 0000000..a7226e5 --- /dev/null +++ b/man/load_flux2_vae_encoder.Rd @@ -0,0 +1,29 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{load_flux2_vae_encoder} +\alias{load_flux2_vae_encoder} +\title{Load the FLUX.2 VAE encoder from safetensors} +\usage{ +load_flux2_vae_encoder( + path, + latent_channels = 32L, + block_channels = c(128L, 256L, 512L, 512L), + layers_per_block = 2L, + norm_groups = 32L, + verbose = TRUE +) +} +\arguments{ +\item{path}{Path to the VAE .safetensors file (or a directory +containing diffusion_pytorch_model.safetensors).} + +\item{verbose}{Logical.} + +\item{latent_channels,block_channels,layers_per_block,norm_groups}{Constructor arguments for \code{\link{flux2_vae_encoder}}.} +} +\value{ +The loaded \code{flux2_vae_encoder} in eval mode. +} +\description{ +Loads the encoder half plus quant_conv and the BatchNorm running +statistics; decoder and post_quant_conv keys are skipped. +} diff --git a/tools/gen_fixtures_flux2_vae_encoder.py b/tools/gen_fixtures_flux2_vae_encoder.py new file mode 100644 index 0000000..eeebb96 --- /dev/null +++ b/tools/gen_fixtures_flux2_vae_encoder.py @@ -0,0 +1,52 @@ +# Generate FLUX.2 VAE encode parity fixtures for the R port. +# +# Loads the tiny random-init AutoencoderKLFlux2 already checked in by +# gen_fixtures_flux2_vae.py (same config) and encodes a random image the +# way Flux2KleinPipeline does for a reference image: posterior mean +# (sample_mode="argmax"), patchify, BatchNorm normalize, pack. Run via +# tools/gen_fixtures.sh. + +import os +import sys + +import torch +from safetensors.torch import load_file, save_file + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "ref", "upstream", "diffusers", "src")) + +from diffusers import AutoencoderKLFlux2 # noqa: E402 +from diffusers.pipelines.flux2.pipeline_flux2_klein import Flux2KleinPipeline # noqa: E402 + +OUT_DIR = os.path.join(os.path.dirname(__file__), "..", "inst", "tinytest", "fixtures") + +vae = AutoencoderKLFlux2( + in_channels=3, + out_channels=3, + down_block_types=("DownEncoderBlock2D",) * 4, + up_block_types=("UpDecoderBlock2D",) * 4, + block_out_channels=(8, 16, 32, 32), + layers_per_block=2, + latent_channels=32, + norm_num_groups=8, + use_quant_conv=True, + use_post_quant_conv=True, + sample_size=64, +) +vae.load_state_dict(load_file(os.path.join(OUT_DIR, "vae_flux2_tiny.safetensors"))) +vae.eval() + +torch.manual_seed(52) +image = torch.rand(1, 3, 32, 48) * 2 - 1 +with torch.no_grad(): + mean = vae.encode(image).latent_dist.mode() + lat = Flux2KleinPipeline._patchify_latents(mean) + bn_mean = vae.bn.running_mean.view(1, -1, 1, 1) + bn_std = torch.sqrt(vae.bn.running_var.view(1, -1, 1, 1) + vae.config.batch_norm_eps) + tokens = Flux2KleinPipeline._pack_latents((lat - bn_mean) / bn_std) + ids = Flux2KleinPipeline._prepare_image_ids([lat])[0].float() + +io = {"image": image, "mean": mean, "tokens": tokens, "image_ids": ids} +io = {k: v.contiguous() for k, v in io.items()} +save_file(io, os.path.join(OUT_DIR, "vae_flux2_enc_io.safetensors"), + metadata={"purpose": "diffuseR FLUX.2 test fixture"}) +print({k: tuple(v.shape) for k, v in io.items()})