From ed2387679a028ef83652a66cd3ba1ab9efad0939 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Mon, 2 Nov 2026 21:57:54 -0600 Subject: [PATCH 1/2] LTX keyframes: condition on latents, images or frames at any frame ltx23_keyframe(x, frame, strength) and txt2vid_ltx2(keyframes =). At frame 0 a keyframe replaces the first latent frames; elsewhere it is appended as extra tokens positioned at its pixel frame and dropped before decoding. Strength below 1 gives the tokens timestep (1 - s) * sigma and blends each step's x0 estimate back toward the keyframe. image, condition_video and condition_latents become a frame-0 keyframe with bit-identical output. Ported from diffusers' pipeline_ltx2_condition.py (Apache-2.0). --- NAMESPACE | 2 + R/keyframes_ltx23.R | 194 +++++++++++++++++++++++++++ R/txt2vid_ltx23.R | 188 +++++++++++++++++--------- inst/REFERENCES.md | 1 + inst/tinytest/test_keyframes_ltx23.R | 153 +++++++++++++++++++++ man/dot-ltx23_keyframe_state.Rd | 47 +++++++ man/ltx23_keyframe.Rd | 37 +++++ man/txt2vid_ltx2.Rd | 8 ++ 8 files changed, 569 insertions(+), 61 deletions(-) create mode 100644 R/keyframes_ltx23.R create mode 100644 inst/tinytest/test_keyframes_ltx23.R create mode 100644 man/dot-ltx23_keyframe_state.Rd create mode 100644 man/ltx23_keyframe.Rd diff --git a/NAMESPACE b/NAMESPACE index baccd7e..fc5353d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -149,6 +149,7 @@ export(ltx23_fp8_linear) export(ltx23_get_timestep_embedding) export(ltx23_is_fp8_cast_key) export(ltx23_kaiser_sinc_filter1d) +export(ltx23_keyframe) export(ltx23_latent_upsampler) export(ltx23_load_group) export(ltx23_load_pipeline) @@ -278,6 +279,7 @@ S3method(print,bpe_tokenizer) S3method(print,diffuseR_resident) S3method(print,gemma4_tokenizer) S3method(print,ltx23_checkpoint) +S3method(print,ltx23_keyframe) S3method(print,qwen_tokenizer) S3method(print,unigram_tokenizer) diff --git a/R/keyframes_ltx23.R b/R/keyframes_ltx23.R new file mode 100644 index 0000000..bf06a70 --- /dev/null +++ b/R/keyframes_ltx23.R @@ -0,0 +1,194 @@ +# LTX keyframes: content pinned at any frame of a generation. +# +# Ported from diffusers `pipelines/ltx2/pipeline_ltx2_condition.py` +# (Apache-2.0; LTX2VideoCondition, apply_first_frame_conditioning, +# _prepare_keyframe_coords, prepare_latents): +# +# frame 0 the keyframe's latent frames replace the generation's own +# from latent 0 (image-to-video, a carried tail). +# frame f the keyframe's latent tokens are appended to the sequence +# with RoPE positions starting at pixel frame f (no causal +# first-frame fix; a single image spans [f, f + 1)); the +# model attends to them and they are dropped before decoding. +# +# Strength s gives each token a conditioning mask s: it starts at +# noise * (1 - s) + clean * s, sees timestep (1 - s) * sigma, and after +# each step its x0 estimate is blended back toward the keyframe by s. +# Strength 1 pins the content exactly. + +#' Define an LTX keyframe +#' +#' A keyframe conditions an \code{\link{txt2vid_ltx2}} generation on +#' content at a frame: a first frame, an end frame, a carried tail from +#' a previous chunk, or any latent frames in between. Pass a list of +#' them as \code{txt2vid_ltx2(keyframes = )}. +#' +#' @param x The content: normalized video latents [1, 128, k, H/32, +#' W/32] (e.g. from \code{\link{ltx23_tail_latents}} or +#' \code{\link{ltx23_encode_video_frames}}); an image path; an [H, W, +#' 3] image array; or an [F, H, W, 3] array of frames in [0, 1] +#' (VAE-encoded at generation time, so F should be 8k + 1). +#' @param frame Integer. Pixel frame (0-based) the keyframe starts at. +#' @param strength Numeric in (0, 1]. 1 holds the content as given; +#' lower values let the generation move away from it. +#' +#' @details At frame 0 the keyframe replaces the generation's first +#' latent frames and, at strength 1, decodes exactly as given there. +#' Anywhere else it is appended as extra tokens positioned at +#' \code{frame}: the generation attends to them and moves toward them, +#' so they act as a strong guide rather than a copy (at 512x512, a still +#' at frame 48 came back at about 25 dB PSNR). The last latent frame of +#' an 8k + 1 clip starts at pixel frame \code{num_frames - 8}. +#' +#' @return An \code{ltx23_keyframe} object. +#' @export +ltx23_keyframe <- function(x, frame = 0L, strength = 1) { + frame <- as.integer(frame) + if (length(frame) != 1L || is.na(frame) || frame < 0L) { + stop("frame must be a single non-negative integer") + } + if (!is.numeric(strength) || length(strength) != 1L || + is.na(strength) || strength <= 0 || strength > 1) { + stop("strength must be in (0, 1]") + } + ok <- inherits(x, "torch_tensor") || + (is.character(x) && length(x) == 1L) || + (is.numeric(x) && length(dim(x)) %in% c(3L, 4L)) + if (!ok) { + stop("x must be latents, an image path, or an image/frames array") + } + structure(list(x = x, frame = frame, strength = as.numeric(strength)), + class = "ltx23_keyframe") +} + +#' @export +print.ltx23_keyframe <- function(x, ...) { + what <- if (inherits(x$x, "torch_tensor")) { + sprintf("latents [%s]", paste(x$x$shape, collapse = ", ")) + } else if (is.character(x$x)) { + basename(x$x) + } else { + sprintf("array [%s]", paste(dim(x$x), collapse = ", ")) + } + cat(sprintf(" %s at frame %d, strength %g\n", what, + x$frame, x$strength)) + invisible(x) +} + +# A keyframe's content as normalized latents [1, 128, k, lh, lw] on +# `device`, plus how many pixel frames it came from (NA for latents). +# `encode` is called with preprocessed [1, 3, F, H, W] frames only when +# needed. +.ltx23_keyframe_latents <- function(kf, encode, height, width, lh, lw, + device) { + x <- kf$x + if (inherits(x, "torch_tensor")) { + lat <- x$to(device = device, dtype = torch::torch_float32()) + pixel_frames <- NA_integer_ + } else { + frames <- ltx23_preprocess_frames(x, height, width) + pixel_frames <- frames$shape[3] + lat <- encode(frames)$to(device = device, + dtype = torch::torch_float32()) + } + if (lat$ndim != 5L || lat$shape[1] != 1L || lat$shape[2] != 128L || + lat$shape[4] != lh || lat$shape[5] != lw) { + stop(sprintf("keyframe latents must be [1, 128, k, %d, %d] for %dx%d generation; got [%s]", + lh, lw, width, height, paste(lat$shape, collapse = ", "))) + } + list(latents = lat, pixel_frames = pixel_frames) +} + +# RoPE coordinates [1, 3, k*lh*lw, 2] (seconds, pixels) for k appended +# keyframe latent frames starting at pixel frame `frame`: the generation's +# own grid scaled to pixels with no causal fix, offset by `frame`, and +# narrowed to [frame, frame + 1) for a single image (diffusers +# _prepare_keyframe_coords). +.ltx23_keyframe_coords <- function(rope, k, lh, lw, frame, single, fps, + device) { + coords <- rope$prepare_video_coords(1L, k, lh, lw, device, fps = fps) + st <- rope$scale_factors[1] + start <- torch::torch_arange(0, k - 1L, dtype = torch::torch_float32(), + device = device)$repeat_interleave(lh * lw) * + st + frame + end <- if (single) start + 1 else start + st + coords[1, 1, , 1] <- start / fps + coords[1, 1, , 2] <- end / fps + coords +} + +#' Build the conditioned starting state from keyframes +#' +#' @param keyframes List of \code{\link{ltx23_keyframe}} objects. +#' @param latent_frames,latent_height,latent_width Latent geometry. +#' @param noise Standard noise [1, 128, F', H', W']. +#' @param encode Function taking preprocessed frames to latents (only +#' called for pixel keyframes). +#' @param height,width Pixel size of the generation. +#' @param rope The transformer's video RoPE (for appended coordinates). +#' @param fps Frame rate. +#' @param cond_noise_scale Partial re-noising of fully pinned frame-0 +#' tokens (the \code{txt2vid_ltx2} argument of that name). +#' @return list(latents [1, S + A, 128], conditioning_mask [1, S + A] +#' strengths (0 = free), clean [1, S + A, 128], extra_coords [1, 3, A, +#' 2] or NULL, n_main = S). +#' @keywords internal +.ltx23_keyframe_state <- function(keyframes, latent_frames, latent_height, + latent_width, noise, encode, height, width, + rope, fps, cond_noise_scale = 0) { + device <- noise$device + f32 <- torch::torch_float32() + clean <- torch::torch_zeros_like(noise) + strength <- torch::torch_zeros(1L, 1L, latent_frames, latent_height, + latent_width, dtype = f32, device = device) + extra_lat <- list() + extra_str <- list() + extra_coords <- list() + for (kf in keyframes) { + enc <- .ltx23_keyframe_latents(kf, encode, height, width, + latent_height, latent_width, device) + k <- enc$latents$shape[3] + if (kf$frame == 0L) { + if (k > latent_frames) { + stop(sprintf("keyframe at frame 0 has %d latent frames; the generation has %d", + k, latent_frames)) + } + clean$narrow(3L, 1L, k)$copy_(enc$latents) + strength$narrow(3L, 1L, k)$fill_(kf$strength) + } else { + extra_lat[[length(extra_lat) + 1L]] <- enc$latents + extra_str[[length(extra_str) + 1L]] <- torch::torch_full( + c(1L, k * latent_height * latent_width), kf$strength, + dtype = f32, device = device) + extra_coords[[length(extra_coords) + 1L]] <- .ltx23_keyframe_coords( + rope, k, latent_height, latent_width, kf$frame, + identical(enc$pixel_frames, 1L), fps, device) + } + } + latents <- clean * strength + noise * strength$neg()$add(1) + if (cond_noise_scale > 0) { + pinned <- (strength == 1)$to(dtype = f32) + scale <- pinned * cond_noise_scale + latents <- noise * scale + latents * (1 - scale) + } + latents <- ltx23_pack_video_latents(latents) + clean_p <- ltx23_pack_video_latents(clean) + mask <- ltx23_pack_video_latents(strength)$squeeze(3L) + n_main <- latents$shape[2] + extra <- NULL + if (length(extra_lat)) { + e_clean <- torch::torch_cat(lapply(extra_lat, ltx23_pack_video_latents), + dim = 2L) + e_str <- torch::torch_cat(extra_str, dim = 2L) + e_s <- e_str$unsqueeze(3L) + e_noise <- torch::torch_randn_like(e_clean) + latents <- torch::torch_cat(list(latents, + e_clean * e_s + e_noise * (1 - e_s)), + dim = 2L) + clean_p <- torch::torch_cat(list(clean_p, e_clean), dim = 2L) + mask <- torch::torch_cat(list(mask, e_str), dim = 2L) + extra <- torch::torch_cat(extra_coords, dim = 3L) + } + list(latents = latents, conditioning_mask = mask, clean = clean_p, + extra_coords = extra, n_main = n_main) +} diff --git a/R/txt2vid_ltx23.R b/R/txt2vid_ltx23.R index 9bf8339..55a93ff 100644 --- a/R/txt2vid_ltx23.R +++ b/R/txt2vid_ltx23.R @@ -154,7 +154,8 @@ ltx23_unpack_audio_latents <- function(latents, num_mel_bins) { compute_dtype, verbose = TRUE, stage = "", conditioning_mask = NULL, audio_conditioned = FALSE, eta = 0, - noise_seed = NULL) { + noise_seed = NULL, clean_latents = NULL, + extra_video_coords = NULL) { f32 <- torch::torch_float32() if (eta > 0 && !is.null(noise_seed)) { torch::local_torch_manual_seed(noise_seed) @@ -162,28 +163,45 @@ ltx23_unpack_audio_latents <- function(latents, num_mel_bins) { step_noise <- function(like) { if (eta > 0) torch::torch_randn_like(like) else NULL } - keep_mask <- if (!is.null(conditioning_mask)) { - conditioning_mask$unsqueeze(3L) # [B, S, 1] for latent blending - } else { - NULL - } t_zero <- if (audio_conditioned) { torch::torch_tensor(0, device = device, dtype = f32) } else { NULL } - # Compact per-token timestep: the conditioned prefix sees exactly - # two values (t and 0), so the transformer gets a 2-entry timestep - # plus a per-token selector instead of a [B, S] vector -- the - # modulation tensors stay [B, 2, ...] until selected on demand - cond_idx <- if (!is.null(conditioning_mask)) { - conditioning_mask$squeeze(1L)$to(dtype = torch::torch_long()) - } else { - NULL + # Compact per-token timestep: tokens see only a few values (t for + # free tokens, 0 for pinned ones, m * t for a keyframe of strength + # 1 - m), so the transformer gets one entry per value plus a + # per-token selector instead of a [B, S] vector -- the modulation + # tensors stay [B, V, ...] until selected on demand + keep_mask <- NULL + denoise_m <- NULL + cond_idx <- NULL + t_scales <- 1 + if (!is.null(conditioning_mask)) { + m_vals <- 1 - as.numeric(as.array(conditioning_mask$cpu())) + t_scales <- unique(c(1, 0, sort(unique(m_vals)))) + cond_idx <- torch::torch_tensor(match(m_vals, t_scales) - 1L, + dtype = torch::torch_long(), + device = device) + if (all(m_vals %in% c(0, 1))) { + keep_mask <- conditioning_mask$unsqueeze(3L) # [B, S, 1] + } else { + if (is.null(clean_latents)) { + stop("fractional keyframe strength needs clean_latents") + } + denoise_m <- conditioning_mask$neg()$add(1)$unsqueeze(3L) + pinned <- (denoise_m == 0)$to(dtype = f32) + } } video_coords <- transformer$rope$prepare_video_coords(latents$shape[1], latent_frames, latent_height, latent_width, device, fps = frame_rate) + if (!is.null(extra_video_coords)) { + # Guide keyframes: appended tokens at their own positions + video_coords <- torch::torch_cat(list(video_coords, + extra_video_coords$to(dtype = video_coords$dtype)), + dim = 3L) + } audio_coords <- transformer$audio_rope$prepare_audio_coords( audio_latents$shape[1], audio_num_frames, device ) @@ -207,8 +225,8 @@ ltx23_unpack_audio_latents <- function(latents, num_mel_bins) { t_video <- if (is.null(conditioning_mask)) { t } else { - # [t, 0]: index 0 = free tokens, index 1 = conditioned - torch::torch_tensor(c(sigma * scale_mult, 0), + # [t, 0, m * t, ...]: index 0 = free tokens, 1 = pinned + torch::torch_tensor(sigma * scale_mult * t_scales, device = device, dtype = f32) } @@ -236,13 +254,25 @@ ltx23_unpack_audio_latents <- function(latents, num_mel_bins) { # Sampler step in float32 (latents are float32 throughout) sigma_next <- sigmas[i + 1L] - stepped <- .ltx23_step(latents, out$sample$to(dtype = f32), sigma, + velocity <- out$sample$to(dtype = f32) + if (!is.null(denoise_m)) { + # Partial keyframes (reference post_process_latent): each + # token's x0 estimate, at its own timestep m * sigma, is + # pulled toward the keyframe by (1 - m), and the step + # runs on the global sigma from there + x0 <- latents - velocity * denoise_m * sigma + x0 <- x0 * denoise_m + clean_latents * (1 - denoise_m) + velocity <- (latents - x0) / sigma + } + stepped <- .ltx23_step(latents, velocity, sigma, sigma_next, eta, step_noise(latents)) - latents <- if (is.null(keep_mask)) { - stepped - } else { + latents <- if (!is.null(keep_mask)) { # Conditioned tokens stay clean latents * keep_mask + stepped * (1 - keep_mask) + } else if (!is.null(denoise_m)) { + stepped * (1 - pinned) + clean_latents * pinned + } else { + stepped } if (!audio_conditioned) { audio_latents <- .ltx23_step(audio_latents, @@ -792,6 +822,12 @@ ltx23_text_embeds <- function(prompt, pipeline, text_encoder = NULL, #' width/32] (e.g. \code{\link{ltx23_tail_latents}} on a previous #' result), used directly as the frozen prefix with no VAE encode. #' Mutually exclusive with \code{image} and \code{condition_video}. +#' @param keyframes Optional list of \code{\link{ltx23_keyframe}} +#' objects: latents, images or frames at any frame and strength. At +#' frame 0 they replace the first latent frames; elsewhere they are +#' appended as extra tokens at their position. \code{image}, +#' \code{condition_video} and \code{condition_latents} are a frame-0 +#' keyframe and may be combined with these. #' @param resident Character vector of pipeline component names #' ("transformer", "vae", "audio_vae", "connectors", "vocoder") to #' keep on the compute device after their phase instead of @@ -837,7 +873,8 @@ txt2vid_ltx2 <- function(prompt, pipeline, text_encoder = NULL, tone_map_compression = 0, phase_offload = TRUE, image = NULL, condition_video = NULL, conditioning_frames = 9L, cond_noise_scale = 0, - condition_latents = NULL, resident = character(), + condition_latents = NULL, keyframes = NULL, + resident = character(), trim_frames = 0L, audio = NULL, verbose = TRUE, sampler = c("auto", "euler", "euler_ancestral")) { level <- .verbosity(verbose) @@ -859,7 +896,13 @@ txt2vid_ltx2 <- function(prompt, pipeline, text_encoder = NULL, if (n_sources > 1L) { stop("Provide only one of image (i2v), condition_video, or condition_latents.") } - conditioned <- n_sources > 0L + if (inherits(keyframes, "ltx23_keyframe")) { + keyframes <- list(keyframes) + } + if (!all(vapply(keyframes, inherits, logical(1), "ltx23_keyframe"))) { + stop("keyframes must be a list of ltx23_keyframe() objects") + } + conditioned <- n_sources > 0L || length(keyframes) > 0L known_components <- c("transformer", "vae", "audio_vae", "connectors", "vocoder") if (length(resident) && !all(resident %in% known_components)) { @@ -871,7 +914,8 @@ txt2vid_ltx2 <- function(prompt, pipeline, text_encoder = NULL, stop("trim_frames must be in [0, num_frames)") } if (conditioned && two_stage) { - stop("Prefix conditioning with two_stage = TRUE is not supported yet.") + stop("Conditioning (image, condition_video, condition_latents or ", + "keyframes) with two_stage = TRUE is not supported yet.") } if (two_stage) { if (is.null(upsampler)) { @@ -1041,40 +1085,31 @@ txt2vid_ltx2 <- function(prompt, pipeline, text_encoder = NULL, audio_num_frames <- as.integer(round(num_frames / frame_rate * 25)) latent_mel_bins <- 16L # 64 mel bins / 4 - # Prefix conditioning: encode the start image (i2v) or the tail of - # a previous clip (continuation) into normalized latents - cond_latents <- NULL - if (!is.null(condition_latents)) { - # Already-encoded prefix (e.g. ltx23_tail_latents of a previous - # chunk): no VAE touch at all - cond_latents <- condition_latents$to(device = device, dtype = f32) - if (cond_latents$ndim != 5L || cond_latents$shape[2] != 128L || - cond_latents$shape[4] != s1_height || - cond_latents$shape[5] != s1_width) { - stop(sprintf( - "condition_latents must be [1, 128, k, %d, %d] for %dx%d generation", - s1_height, s1_width, width, height)) - } - } else if (conditioned) { - vae <- onload("vae") - cond_frames <- if (!is.null(image)) { - ltx23_preprocess_frames(image, height, width) + # Conditioning is a list of keyframes. The older single-source + # arguments are a replace keyframe at frame 0: a start image (i2v), + # the tail of a previous clip, or already-encoded latents (e.g. + # ltx23_tail_latents of a previous chunk, no VAE touch at all) + if (!is.null(condition_latents) && + (condition_latents$ndim != 5L || condition_latents$shape[2] != 128L || + condition_latents$shape[4] != s1_height || + condition_latents$shape[5] != s1_width)) { + stop(sprintf( + "condition_latents must be [1, 128, k, %d, %d] for %dx%d generation", + s1_height, s1_width, width, height)) + } + legacy <- if (!is.null(condition_latents)) { + condition_latents + } else if (!is.null(image)) { + image + } else if (!is.null(condition_video)) { + if (is.character(condition_video)) { + ltx23_read_tail_frames(condition_video, conditioning_frames) } else { - arr <- if (is.character(condition_video)) { - ltx23_read_tail_frames(condition_video, conditioning_frames) - } else { - condition_video - } - ltx23_preprocess_frames(arr, height, width) - } - if (verbose) { - message(sprintf("Encoding %d conditioning frame(s)...", - cond_frames$shape[3])) + condition_video } - cond_latents <- ltx23_encode_video_frames(vae, cond_frames)$ - to(device = device) - offload("vae") - gc(verbose = FALSE) + } + if (!is.null(legacy)) { + keyframes <- c(list(ltx23_keyframe(legacy, frame = 0L)), keyframes) } noise <- torch::torch_randn( @@ -1082,14 +1117,39 @@ txt2vid_ltx2 <- function(prompt, pipeline, text_encoder = NULL, device = device, dtype = f32 ) conditioning_mask <- NULL + clean_latents <- NULL + extra_coords <- NULL + n_main <- NULL if (conditioned) { - prep <- ltx23_prepare_conditioned_latents( - cond_latents, latent_frames, s1_height, s1_width, - noise, cond_noise_scale = cond_noise_scale - ) + vae_on <- FALSE + encode <- function(frames) { + if (!vae_on) { + vae <<- onload("vae") + vae_on <<- TRUE + } + if (verbose) { + message(sprintf("Encoding %d conditioning frame(s)...", + frames$shape[3])) + } + ltx23_encode_video_frames(vae, frames) + } + vae <- NULL + rope <- pipeline$transformer$rope + prep <- torch::with_no_grad(.ltx23_keyframe_state( + keyframes, latent_frames, s1_height, s1_width, noise, encode, + height, width, rope, frame_rate, + cond_noise_scale = cond_noise_scale + )) + if (vae_on) { + offload("vae") + gc(verbose = FALSE) + } latents <- prep$latents conditioning_mask <- prep$conditioning_mask - rm(prep, cond_latents) + clean_latents <- prep$clean + extra_coords <- prep$extra_coords + n_main <- prep$n_main + rm(prep) } else { latents <- ltx23_pack_video_latents(noise) } @@ -1140,9 +1200,15 @@ txt2vid_ltx2 <- function(prompt, pipeline, text_encoder = NULL, conditioning_mask = conditioning_mask, audio_conditioned = audio_conditioned, eta = eta, - noise_seed = if (!is.null(seed)) seed + 10000 + noise_seed = if (!is.null(seed)) seed + 10000, + clean_latents = clean_latents, + extra_video_coords = extra_coords ) latents <- denoised$latents + if (!is.null(n_main) && latents$shape[2] > n_main) { + # Drop appended guide-keyframe tokens before decoding + latents <- latents$narrow(2L, 1L, n_main) + } audio_latents <- denoised$audio_latents if (two_stage) { diff --git a/inst/REFERENCES.md b/inst/REFERENCES.md index e872229..d539f71 100644 --- a/inst/REFERENCES.md +++ b/inst/REFERENCES.md @@ -17,6 +17,7 @@ module; this file documents the actual lineage, idea by idea. | LTX-2.5 transformer flags (`ff_bias = FALSE` video FFN, `keyframes_abs_pos_embedding` after `proj_in`) | diffusers `models/transformers/transformer_ltx2.py` (Apache-2.0); flag values from the embedded config of the official LTX-2.5 checkpoints (format facts) | | Gemma 4 text encoder (LTX-2.5) | Ported from HuggingFace **transformers** (Apache-2.0) `models/gemma4_unified/modeling_gemma4_unified.py` (text stack) and `modeling_rope_utils.py` (`proportional` RoPE); the LTX encode recipe (strip, left pad, all 49 hidden states stacked) from diffusers `modular_pipelines/ltx2/encoders.py` (Apache-2.0). Parity fixtures: `tools/gen_fixtures_gemma4.py` | | Gemma 4 tokenizer | Re-implements the HuggingFace **tokenizers** BPE pipeline described by the shipped `tokenizer.json` (normalizer, byte fallback, ranked merges, post-processor template); verified against `tokenizers` ids | +| Keyframes (`ltx23_keyframe()`, `txt2vid_ltx2(keyframes =)`): frame-0 replacement, keyframes elsewhere appended as extra tokens at their pixel position (no causal fix, single image spans one frame), per-token strength as conditioning mask with timestep `(1 - s) * sigma` and the post-step x0 blend toward the clean keyframe | Ported from diffusers `pipelines/ltx2/pipeline_ltx2_condition.py` (Apache-2.0): `apply_first_frame_conditioning`, `_prepare_keyframe_coords`, `prepare_latents` and its denoising loop | | Ancestral (SDE) Euler step for LTX-2.5, eta 1 | Ported from diffusers `schedulers/scheduling_ltx_euler_ancestral_rf.py` (Apache-2.0). That LTX-2.5+ checkpoints use it, and the stage noise seeds `seed + 10000` / `seed + 20000`, are numeric facts from Lightricks' LTX-2 release (`ANCESTRAL_SAMPLER_SINCE_VERSION`, LTX-2 Community License; no code taken) | ## FLUX.1-schnell diff --git a/inst/tinytest/test_keyframes_ltx23.R b/inst/tinytest/test_keyframes_ltx23.R new file mode 100644 index 0000000..eef54bd --- /dev/null +++ b/inst/tinytest/test_keyframes_ltx23.R @@ -0,0 +1,153 @@ +# General LTX keyframes: frame-0 replacement, appended keyframes at any +# other frame, strengths, and the older single-source arguments as a +# frame-0 keyframe. Tiny random-weight pipeline on the CPU, as in +# test_txt2vid_ltx23.R. + +if (!requireNamespace("torch", quietly = TRUE) || !torch::torch_is_installed()) { + exit_file("torch not fully installed") +} + +library(diffuseR) + +# ---- constructor ----------------------------------------------------------- +lat <- torch::torch_randn(1L, 128L, 1L, 2L, 2L) +kf <- ltx23_keyframe(lat, frame = 16L, strength = 0.5) +expect_inherits(kf, "ltx23_keyframe") +expect_equal(kf$frame, 16L) +expect_equal(kf$strength, 0.5) +expect_error(ltx23_keyframe(lat, frame = -1L), "non-negative") +expect_error(ltx23_keyframe(lat, strength = 0), "strength") +expect_error(ltx23_keyframe(lat, strength = 1.5), "strength") +expect_error(ltx23_keyframe(list()), "latents, an image path") + +# ---- tiny pipeline ------------------------------------------------------- +torch::torch_manual_seed(99) +transformer <- ltx23_transformer( + in_channels = 128L, out_channels = 128L, + num_attention_heads = 2L, attention_head_dim = 8L, + cross_attention_dim = 16L, + audio_in_channels = 128L, audio_out_channels = 128L, + audio_num_attention_heads = 2L, audio_attention_head_dim = 4L, + audio_cross_attention_dim = 8L, + num_layers = 1L +) +transformer$eval() +connectors <- ltx23_text_connectors( + caption_channels = 8L, text_proj_in_factor = 3L, + video_connector_num_attention_heads = 2L, + video_connector_attention_head_dim = 8L, + video_connector_num_layers = 1L, + video_connector_num_learnable_registers = 4L, + audio_connector_num_attention_heads = 2L, + audio_connector_attention_head_dim = 4L, + audio_connector_num_layers = 1L, + audio_connector_num_learnable_registers = 4L, + video_hidden_dim = 16L, audio_hidden_dim = 8L +) +connectors$eval() +src <- readLines(system.file("tinytest", "test_txt2vid_ltx23.R", + package = "diffuseR")) +if (!length(src)) { + src <- readLines("test_txt2vid_ltx23.R") +} +# The VAE, vocoder, pipeline and stub embeds exactly as the pipeline +# test builds them +setup <- src[grep("^vae <- ltx23_video_vae", src):(grep("^res <- txt2vid_ltx2", src) - 1L)] +eval(parse(text = setup)) + +run <- function(...) { + txt2vid_ltx2("x", pipe, prompt_embeds = stub_embeds, width = 64L, + height = 64L, num_frames = 25L, frame_rate = 24, seed = 7L, + device = "cpu", dtype = "float32", decode_audio = FALSE, + verbose = FALSE, ...) +} +lat_of <- function(res) { + diffuseR:::ltx23_unpack_video_latents(res$latents, res$latent_shape[1], + res$latent_shape[2], + res$latent_shape[3]) +} +torch::torch_manual_seed(5) +k2 <- torch::torch_randn(1L, 128L, 2L, 2L, 2L) +k1 <- torch::torch_randn(1L, 128L, 1L, 2L, 2L) + +# ---- a frame-0 replace keyframe is condition_latents --------------------- +a <- run(condition_latents = k2) +b <- run(keyframes = list(ltx23_keyframe(k2, frame = 0L))) +expect_equal(max(abs(a$video - b$video)), 0) +# a single keyframe need not be wrapped in a list +c1 <- run(keyframes = ltx23_keyframe(k2, frame = 0L)) +expect_equal(max(abs(a$video - c1$video)), 0) + +# ---- the frame-0 keyframe stays exactly as given --------------------------- +# 25 frames -> 4 latent frames; the last starts at pixel frame 17 +both <- run(keyframes = list(ltx23_keyframe(k1, frame = 0L), + ltx23_keyframe(k1 * 2, frame = 17L))) +lb <- lat_of(both) +expect_equal(as.numeric((lb$narrow(3L, 1L, 1L) - k1)$abs()$max()), 0) +expect_equal(both$latent_shape, c(4L, 2L, 2L)) +expect_equal(dim(both$video), c(25L, 64L, 64L, 3L)) +expect_true(max(abs(both$video - run(keyframes = ltx23_keyframe(k1))$video)) > 1e-6) +# a frame-0 keyframe longer than the generation is refused +expect_error(run(keyframes = ltx23_keyframe(torch::torch_randn(1L, 128L, 5L, 2L, 2L))), + "has 5 latent frames") + +# ---- the starting state: replaced tokens, appended tokens, masks ----------- +noise <- torch::torch_randn(1L, 128L, 4L, 2L, 2L) +enc_fail <- function(frames) stop("no pixel keyframes here") +st <- diffuseR:::.ltx23_keyframe_state( + list(ltx23_keyframe(k1, 0L), ltx23_keyframe(k2, 9L, strength = 0.5)), + 4L, 2L, 2L, noise, enc_fail, 64L, 64L, transformer$rope, 24) +# 16 main tokens (4 x 2 x 2), then 8 appended (2 latent frames x 2 x 2) +expect_equal(st$n_main, 16L) +expect_equal(as.integer(st$latents$shape), c(1L, 24L, 128L)) +expect_equal(as.numeric(st$conditioning_mask), c(rep(1, 4), rep(0, 12), rep(0.5, 8))) +expect_equal(as.integer(st$extra_coords$shape), c(1L, 3L, 8L, 2L)) +# frame-0 tokens start clean; free tokens are pure noise; appended tokens +# start halfway between their keyframe and noise +packed <- diffuseR:::ltx23_pack_video_latents +expect_equal(as.numeric((st$latents[, 1:4, ] - packed(k1))$abs()$max()), 0) +expect_equal(as.numeric((st$latents[, 5:16, ] - packed(noise)[, 5:16, ])$abs()$max()), 0) +expect_equal(as.numeric((st$clean[, 17:24, ] - packed(k2))$abs()$max()), 0) + +# ---- appended keyframes: same output geometry, effect on the video --------- +g <- run(keyframes = list(ltx23_keyframe(k1, frame = 0L), + ltx23_keyframe(k1 * 2, frame = 12L))) +expect_equal(dim(g$video), c(25L, 64L, 64L, 3L)) +expect_equal(g$latent_shape, c(4L, 2L, 2L)) +expect_true(max(abs(g$video - run(keyframes = ltx23_keyframe(k1))$video)) > 1e-6) +gp <- run(keyframes = ltx23_keyframe(k1, frame = 12L, strength = 0.6)) +expect_true(all(is.finite(gp$video))) +# partial strength at frame 0 (per-token timesteps and the x0 blend) +h0 <- run(keyframes = ltx23_keyframe(k1, frame = 0L, strength = 0.5)) +expect_true(all(is.finite(h0$video))) +d_h0 <- as.numeric((lat_of(h0)$narrow(3L, 1L, 1L) - k1)$abs()$mean()) +d_free <- as.numeric((lat_of(run())$narrow(3L, 1L, 1L) - k1)$abs()$mean()) +expect_true(d_h0 > 0) +expect_true(d_h0 < d_free) + +# appended coordinates: [start, end) in seconds offset by the frame, no +# causal first-frame fix, one pixel frame for an image (diffusers +# _prepare_keyframe_coords) +rope <- transformer$rope +co <- diffuseR:::.ltx23_keyframe_coords(rope, 2L, 2L, 2L, 16L, FALSE, 24, + "cpu") +expect_equal(dim(co), c(1L, 3L, 8L, 2L)) +expect_equal(as.numeric(co[1, 1, 1, ]) * 24, c(16, 24), tolerance = 1e-6) +expect_equal(as.numeric(co[1, 1, 5, ]) * 24, c(24, 32), tolerance = 1e-6) +co1 <- diffuseR:::.ltx23_keyframe_coords(rope, 1L, 2L, 2L, 12L, TRUE, 24, + "cpu") +expect_equal(as.numeric(co1[1, 1, 1, ]) * 24, c(12, 13), tolerance = 1e-6) +# spatial axes match the generation's own grid +full <- rope$prepare_video_coords(1L, 2L, 2L, 2L, "cpu", fps = 24) +expect_equal(as.numeric((co[, 2:3, , ] - full[, 2:3, , ])$abs()$max()), 0) + +# ---- pixel keyframes are encoded at generation time ---------------------- +img <- array(runif(64 * 64 * 3), dim = c(64L, 64L, 3L)) +pi <- run(keyframes = list(ltx23_keyframe(img, frame = 0L), + ltx23_keyframe(img, frame = 17L))) +expect_equal(dim(pi$video), c(25L, 64L, 64L, 3L)) +expect_equal(max(abs(run(image = img)$video - run(keyframes = ltx23_keyframe(img))$video)), 0) + +# ---- still refused with two_stage ---------------------------------------- +expect_error(run(keyframes = ltx23_keyframe(k1), two_stage = TRUE), + "two_stage") diff --git a/man/dot-ltx23_keyframe_state.Rd b/man/dot-ltx23_keyframe_state.Rd new file mode 100644 index 0000000..df09107 --- /dev/null +++ b/man/dot-ltx23_keyframe_state.Rd @@ -0,0 +1,47 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{.ltx23_keyframe_state} +\alias{.ltx23_keyframe_state} +\title{Build the conditioned starting state from keyframes} +\usage{ +.ltx23_keyframe_state( + keyframes, + latent_frames, + latent_height, + latent_width, + noise, + encode, + height, + width, + rope, + fps, + cond_noise_scale = 0 +) +} +\arguments{ +\item{keyframes}{List of \code{\link{ltx23_keyframe}} objects.} + +\item{noise}{Standard noise [1, 128, F', H', W'].} + +\item{encode}{Function taking preprocessed frames to latents (only +called for pixel keyframes).} + +\item{rope}{The transformer's video RoPE (for appended coordinates).} + +\item{fps}{Frame rate.} + +\item{cond_noise_scale}{Partial re-noising of fully pinned frame-0 +tokens (the \code{txt2vid_ltx2} argument of that name).} + +\item{latent_frames,latent_height,latent_width}{Latent geometry.} + +\item{height,width}{Pixel size of the generation.} +} +\value{ +list(latents [1, S + A, 128], conditioning_mask [1, S + A] + strengths (0 = free), clean [1, S + A, 128], extra_coords [1, 3, A, + 2] or NULL, n_main = S). +} +\description{ +Build the conditioned starting state from keyframes +} +\keyword{internal} diff --git a/man/ltx23_keyframe.Rd b/man/ltx23_keyframe.Rd new file mode 100644 index 0000000..edd32d5 --- /dev/null +++ b/man/ltx23_keyframe.Rd @@ -0,0 +1,37 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_keyframe} +\alias{ltx23_keyframe} +\title{Define an LTX keyframe} +\usage{ +ltx23_keyframe(x, frame = 0L, strength = 1) +} +\arguments{ +\item{x}{The content: normalized video latents [1, 128, k, H/32, +W/32] (e.g. from \code{\link{ltx23_tail_latents}} or +\code{\link{ltx23_encode_video_frames}}); an image path; an [H, W, +3] image array; or an [F, H, W, 3] array of frames in [0, 1] +(VAE-encoded at generation time, so F should be 8k + 1).} + +\item{frame}{Integer. Pixel frame (0-based) the keyframe starts at.} + +\item{strength}{Numeric in (0, 1]. 1 holds the content as given; +lower values let the generation move away from it.} +} +\value{ +An \code{ltx23_keyframe} object. +} +\description{ +A keyframe conditions an \code{\link{txt2vid_ltx2}} generation on +content at a frame: a first frame, an end frame, a carried tail from +a previous chunk, or any latent frames in between. Pass a list of +them as \code{txt2vid_ltx2(keyframes = )}. +} +\details{ +At frame 0 the keyframe replaces the generation's first +latent frames and, at strength 1, decodes exactly as given there. +Anywhere else it is appended as extra tokens positioned at +\code{frame}: the generation attends to them and moves toward them, +so they act as a strong guide rather than a copy (at 512x512, a still +at frame 48 came back at about 25 dB PSNR). The last latent frame of +an 8k + 1 clip starts at pixel frame \code{num_frames - 8}. +} diff --git a/man/txt2vid_ltx2.Rd b/man/txt2vid_ltx2.Rd index d7953b7..93e1ed4 100644 --- a/man/txt2vid_ltx2.Rd +++ b/man/txt2vid_ltx2.Rd @@ -33,6 +33,7 @@ txt2vid_ltx2( conditioning_frames = 9L, cond_noise_scale = 0, condition_latents = NULL, + keyframes = NULL, resident = character(), trim_frames = 0L, audio = NULL, @@ -121,6 +122,13 @@ width/32] (e.g. \code{\link{ltx23_tail_latents}} on a previous result), used directly as the frozen prefix with no VAE encode. Mutually exclusive with \code{image} and \code{condition_video}.} +\item{keyframes}{Optional list of \code{\link{ltx23_keyframe}} +objects: latents, images or frames at any frame and strength. At +frame 0 they replace the first latent frames; elsewhere they are +appended as extra tokens at their position. \code{image}, +\code{condition_video} and \code{condition_latents} are a frame-0 +keyframe and may be combined with these.} + \item{resident}{Character vector of pipeline component names ("transformer", "vae", "audio_vae", "connectors", "vocoder") to keep on the compute device after their phase instead of From 9f8a9f0964f595a2baa0b5c10af2f57e2b83e76b Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Mon, 2 Nov 2026 21:57:57 -0600 Subject: [PATCH 2/2] Bump version to 0.2.2.26 --- DESCRIPTION | 2 +- NEWS.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index ccba33d..dc42751 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: diffuseR Title: Functional Interface to Diffusion Models in R -Version: 0.2.2.25 +Version: 0.2.2.26 Authors@R: c( person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"), comment = c(ORCID = "0009-0005-4248-604X")), diff --git a/NEWS.md b/NEWS.md index 4914837..cca39be 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,17 @@ +# diffuseR 0.2.2.26 + +* LTX keyframes: `ltx23_keyframe(x, frame, strength)` and + `txt2vid_ltx2(keyframes = list(...))` condition a generation on + latents, an image or frames at any frame and strength. At frame 0 + they replace the first latent frames (as `image`, `condition_video` + and `condition_latents` always did, which are now frame-0 keyframes); + elsewhere they are appended as extra tokens at their position and + dropped before decoding. Strength below 1 uses per-token timesteps and + pulls each step's estimate toward the keyframe. Ported from diffusers' + LTX-2 condition pipeline. Start/end anchors, a carried tail plus an end + anchor, and mid-clip stills are all keyframe lists. Existing + single-source calls produce bit-identical output. + # diffuseR 0.2.2.25 * AnimateDiff text-to-video on Stable Diffusion 1.5: