From fe23d0a524241745f9f41e7c2c13bbc29f7ea5e0 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Wed, 28 Oct 2026 17:33:14 -0500 Subject: [PATCH 1/3] LTX-2.5 on 8 and 12 GB cards: stream NF4 blocks, connectors and Gemma 4 layers from pinned memory --- CLAUDE.md | 17 ++++++ R/connectors_ltx23.R | 16 ++++- R/dit_ltx23.R | 69 +++++++++++++++++----- R/gemma4_text_encoder.R | 21 ++++++- R/memory_ltx23.R | 45 ++++++++++---- R/quantize_gemma4.R | 14 ++++- R/staging.R | 13 +++++ R/stream_ltx23.R | 84 +++++++++++++++++++++++++++ R/txt2vid_ltx23.R | 71 ++++++++++++++++++---- inst/tinytest/test_connectors_ltx23.R | 24 ++++++++ inst/tinytest/test_dit_ltx23.R | 27 +++++++++ inst/tinytest/test_gemma4_nf4.R | 13 +++++ inst/tinytest/test_memory_ltx23.R | 23 ++++++++ man/load_gemma4_nf4.Rd | 6 ++ man/load_gemma4_text_encoder.Rd | 4 ++ man/ltx23_load_pipeline.Rd | 9 +++ man/ltx23_memory_profile.Rd | 16 ++++- 17 files changed, 423 insertions(+), 49 deletions(-) create mode 100644 R/stream_ltx23.R diff --git a/CLAUDE.md b/CLAUDE.md index 080755f..fe17834 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -495,6 +495,23 @@ params in fp32 before the bf16 cast needs 22 GB of host RAM, so the loader constructs a `.construct_skeleton()` (FLUX 3 opts in via `skeleton = TRUE` in `.flux_family_hooks`). +### LTX on 8 and 12 GB cards + +`ltx23_load_pipeline(resident_gb = )` keeps the leading NF4 blocks on +the GPU and streams the rest from pinned memory one block per +TorchScript call (`.ltx23_block_stream`); under 7 GB the text +connectors stream too. `load_gemma4_nf4(stream = TRUE)` streams Gemma 4 +layers. `ltx23_memory_profile()` picks `resident_gb = vram - 4` below +14 GB. Simulated (ballast tensor) at 768x512x121: 12 GB and 8 GB both +~84 s, same as 16 GB; PCIe 5 hides 6.7 GB/step of streaming. + +Gotcha that cost an hour: the first render in a process differs from +later ones with the same seed (TorchScript's profiling executor swaps +graphs after its first calls; max latent diff ~3), so compare streamed +vs resident only after a warm-up run. Host RAM: the pipeline's pinned +staging (~21 GB) plus Gemma 4 do not fit this host's free RAM in one +process; set `MALLOC_MMAP_THRESHOLD_=1048576` and encode separately. + ### LTX-2.3 Video Generation (clean-room rewrite in progress) The original LTX-2.0 port was removed and is being replaced by a ground-up diff --git a/R/connectors_ltx23.R b/R/connectors_ltx23.R index c9185b9..128c2c6 100644 --- a/R/connectors_ltx23.R +++ b/R/connectors_ltx23.R @@ -240,10 +240,19 @@ ltx23_connector_transformer_1d <- torch::nn_module( rotary_emb <- self$rope(batch_size, seq_len, device = hidden_states$device) + # Block streaming (see .ltx23_connector_stream) + stream <- self$layer_stream for (i in seq_along(self$transformer_blocks)) { + if (!is.null(stream)) { + .staged_onload(stream[[i]], hidden_states$device) + } hidden_states <- self$transformer_blocks[[i]]( hidden_states, attention_mask = attention_mask, rotary_emb = rotary_emb ) + if (!is.null(stream)) { + .staged_offload(stream[[i]]) + gc(verbose = FALSE, full = FALSE) + } } hidden_states <- self$norm_out(hidden_states) @@ -355,8 +364,11 @@ ltx23_text_connectors <- torch::nn_module( video_norm <- norm_states$mul(sqrt(self$video_hidden_dim / self$caption_channels)) audio_norm <- norm_states$mul(sqrt(self$audio_hidden_dim / self$caption_channels)) - video_proj <- self$video_text_proj_in(video_norm) - audio_proj <- self$audio_text_proj_in(audio_norm) + video_proj <- .unit_call(self, "video_text_proj_in", video_norm$device, + function() self$video_text_proj_in(video_norm)) + audio_proj <- .unit_call(self, "audio_text_proj_in", audio_norm$device, + function() self$audio_text_proj_in(audio_norm)) + rm(norm_states, video_norm, audio_norm) # Multiplicative [B, S] mask -> additive [B, 1, 1, S] with -finfo max text_dtype <- video_proj$dtype diff --git a/R/dit_ltx23.R b/R/dit_ltx23.R index fad921d..666fe1a 100644 --- a/R/dit_ltx23.R +++ b/R/dit_ltx23.R @@ -368,23 +368,58 @@ ltx23_transformer <- torch::nn_module( !is.null(video_ca_rotary_emb) && !is.null(audio_ca_rotary_emb) && .ltx23_jit_block_ok(self$transformer_blocks[[1]]) + # Block streaming (see .ltx23_block_stream): blocks past n_resident + # live in pinned host memory and visit the device one at a time + stream <- self$block_stream + n_blocks <- length(self$transformer_blocks) + n_resident <- if (is.null(stream)) n_blocks else stream$n_resident + stream_on <- function(i) { + if (i > n_resident) { + .staged_onload(stream$staging[[i - n_resident]], + hidden_states$device) + } + } + stream_off <- function(i) { + if (i > n_resident) { + .staged_offload(stream$staging[[i - n_resident]]) + # The onload's R-side copies hold the device storage until + # collected + gc(verbose = FALSE, full = FALSE) + } + } + if (use_jit) { - res <- .ltx23_jit_run_stack( - self$transformer_blocks, - hidden_states, audio_hidden_states, - encoder_hidden_states, audio_encoder_hidden_states, - temb, temb_audio, - video_ca_scale_shift, audio_ca_scale_shift, - video_ca_a2v_gate, audio_ca_v2a_gate, - temb_prompt, temb_prompt_audio, - video_rotary_emb, audio_rotary_emb, - video_ca_rotary_emb, audio_ca_rotary_emb, - encoder_attention_mask, - audio_encoder_attention_mask, - cond_token_index = cond_token_index - ) - hidden_states <- res[[1]] - audio_hidden_states <- res[[2]] + run_stack <- function(blocks) { + res <- .ltx23_jit_run_stack( + blocks, + hidden_states, audio_hidden_states, + encoder_hidden_states, audio_encoder_hidden_states, + temb, temb_audio, + video_ca_scale_shift, audio_ca_scale_shift, + video_ca_a2v_gate, audio_ca_v2a_gate, + temb_prompt, temb_prompt_audio, + video_rotary_emb, audio_rotary_emb, + video_ca_rotary_emb, audio_ca_rotary_emb, + encoder_attention_mask, + audio_encoder_attention_mask, + cond_token_index = cond_token_index + ) + hidden_states <<- res[[1]] + audio_hidden_states <<- res[[2]] + } + if (n_resident == n_blocks) { + run_stack(self$transformer_blocks) + } else { + if (n_resident > 0L) { + run_stack(lapply(seq_len(n_resident), + function(i) self$transformer_blocks[[i]])) + } + for (i in seq(n_resident + 1L, n_blocks)) { + stream_on(i) + run_stack(list(self$transformer_blocks[[i]])) + stream_off(i) + } + } } block_gc <- isTRUE(getOption("diffuseR.block_gc")) @@ -402,6 +437,7 @@ ltx23_transformer <- torch::nn_module( eager_blocks <- if (use_jit) integer(0) else seq_along(self$transformer_blocks) for (block_idx in eager_blocks) { is_stg_block <- (block_idx - 1L) %in% stg_blocks # reference indices are 0-based + stream_on(block_idx) res <- self$transformer_blocks[[block_idx]]( hidden_states = hidden_states, audio_hidden_states = audio_hidden_states, @@ -429,6 +465,7 @@ ltx23_transformer <- torch::nn_module( ) hidden_states <- res[[1]] audio_hidden_states <- res[[2]] + stream_off(block_idx) if (isTRUE(getOption("diffuseR.debug"))) { ms <- torch::cuda_memory_stats() message(sprintf(" block %d: %.2f GB allocated, %.2f GB reserved", diff --git a/R/gemma4_text_encoder.R b/R/gemma4_text_encoder.R index 4f12654..c852cbe 100644 --- a/R/gemma4_text_encoder.R +++ b/R/gemma4_text_encoder.R @@ -217,13 +217,23 @@ gemma4_text_model <- torch::nn_module( dtype = dtype) all_hidden <- NULL + # Layer streaming (load with stream = TRUE): each layer's weights + # visit the device only for its own forward + stream <- self$layer_stream for (i in seq_along(self$layers)) { if (output_hidden_states) { all_hidden <- c(all_hidden, list(hidden_states)) } type <- self$config$layer_types[[i]] + if (!is.null(stream)) { + .staged_onload(stream[[i]], device) + } hidden_states <- self$layers[[i]](hidden_states, masks[[type]], rope[[type]][[1]], rope[[type]][[2]]) + if (!is.null(stream)) { + .staged_offload(stream[[i]]) + gc(verbose = FALSE, full = FALSE) + } } hidden_states <- self$norm(hidden_states) if (output_hidden_states) { @@ -342,6 +352,8 @@ gemma4_text_model <- torch::nn_module( #' \code{\link{encode_with_gemma4}} can move the model to the GPU per #' encode at DMA speed. Default follows #' \code{options(diffuseR.pin_staging)}. +#' @param stream Logical. Stage layer by layer (see +#' \code{\link{load_gemma4_nf4}}). #' @param verbose Logical. #' #' @return A \code{gemma4_text_model}. @@ -349,10 +361,10 @@ gemma4_text_model <- torch::nn_module( #' @export load_gemma4_text_encoder <- function(path, device = "cpu", dtype = "bfloat16", pin = getOption("diffuseR.pin_staging", TRUE), - verbose = TRUE) { + stream = FALSE, verbose = TRUE) { if (file.exists(file.path(path, "manifest.json"))) { return(load_gemma4_nf4(path, device = device, dtype = dtype, - pin = pin, verbose = verbose)) + pin = pin, stream = stream, verbose = verbose)) } ckpt <- ltx23_open_checkpoint(path, require_version = NULL) meta <- ckpt$handle$metadata[["__metadata__"]] @@ -383,7 +395,10 @@ load_gemma4_text_encoder <- function(path, device = "cpu", dtype = "bfloat16", model$to(device = device) model$eval() - if (pin && device == "cpu") { + if (stream && device == "cpu") { + attr(model, "staging") <- .stage_layer_stream(model, model$layers, + "layers.", pin) + } else if (pin && device == "cpu") { st <- .pin_component(model) if (!is.null(st)) { attr(model, "staging") <- st diff --git a/R/memory_ltx23.R b/R/memory_ltx23.R index ce74ee6..a9ff152 100644 --- a/R/memory_ltx23.R +++ b/R/memory_ltx23.R @@ -20,9 +20,18 @@ NULL #' (tiled VAE decode, in-place feed-forward GELU, and the default #' \code{diffuseR.attn_budget} of 1.5e8 all required at that size). #' +#' Below 14 GB the NF4 transformer keeps \code{resident_gb} on the GPU +#' and streams its remaining blocks from pinned host memory every step +#' (pass it to \code{\link{ltx23_load_pipeline}}), and below 12 GB the +#' Gemma 4 encoder streams its layers (\code{stream_text}; pass it as +#' \code{stream} to \code{\link{load_gemma4_text_encoder}}). Measured +#' with the rest of a 16 GB card held back: LTX-2.5 768x512x121 renders +#' in ~84 s on both 12 GB (8 GB resident) and 8 GB (4 GB resident), the +#' same as on the full card. +#' #' \describe{ -#' \item{precision "nf4"}{Weights resident on the GPU; fastest steps; -#' about 9 percent weight round-trip error.} +#' \item{precision "nf4"}{Weights resident on the GPU (or partly +#' streamed); fastest steps; about 9 percent weight round-trip error.} #' \item{precision "fp8"}{Weights CPU-resident, streamed per linear; #' near-bf16 quality; each step pays the PCIe transfer.} #' } @@ -30,7 +39,8 @@ NULL #' @param vram_gb Numeric or NULL (auto-detect free VRAM). #' #' @return Named list with device/dtype placement, \code{attn_chunk}, -#' \code{pin_weights}, and resolution caps. +#' \code{pin_weights}, \code{resident_gb}, \code{stream_text}, and +#' resolution caps. #' #' @export ltx23_memory_profile <- function(vram_gb = NULL) { @@ -49,35 +59,44 @@ ltx23_memory_profile <- function(vram_gb = NULL) { "cpu_only" } + # Streamed tiers keep vram_gb - 4 GB of the transformer resident: + # 768x512x121 activations, connectors and decode fit in the rest + resident <- max(0, vram_gb - 4) profiles <- list( high = list(name = "high", device = "cuda", dtype = "bfloat16", precision = "nf4", phase_offload = TRUE, pin_weights = FALSE, attn_chunk = NULL, - text_device = "cpu", max_resolution = c(704L, 1280L), + text_device = "cpu", resident_gb = Inf, + stream_text = FALSE, + max_resolution = c(704L, 1280L), max_frames = 121L), medium = list( name = "medium", device = "cuda", dtype = "bfloat16", - precision = "fp8", + precision = "nf4", phase_offload = TRUE, pin_weights = TRUE, - attn_chunk = 4096L, - text_device = "cpu", - max_resolution = c(576L, 1024L), + attn_chunk = NULL, + text_device = "cuda", + resident_gb = resident, + stream_text = vram_gb < 12, + max_resolution = c(512L, 768L), max_frames = 121L ), low = list( name = "low", device = "cuda", dtype = "bfloat16", - precision = "fp8", + precision = "nf4", phase_offload = TRUE, pin_weights = TRUE, - attn_chunk = 2048L, - text_device = "cpu", + attn_chunk = NULL, + text_device = "cuda", + resident_gb = resident, + stream_text = TRUE, max_resolution = c(512L, 768L), - max_frames = 65L + max_frames = 121L ), cpu_only = list( name = "cpu_only", @@ -88,6 +107,8 @@ ltx23_memory_profile <- function(vram_gb = NULL) { pin_weights = FALSE, attn_chunk = NULL, text_device = "cpu", + resident_gb = Inf, + stream_text = FALSE, max_resolution = c(384L, 640L), max_frames = 33L ) diff --git a/R/quantize_gemma4.R b/R/quantize_gemma4.R index af604c6..bffdbdd 100644 --- a/R/quantize_gemma4.R +++ b/R/quantize_gemma4.R @@ -164,6 +164,10 @@ gemma4_quantize_nf4 <- function(path, output_dir = NULL, shard_bytes = 1.9e9, #' \code{\link{encode_with_gemma4}} can move the model to the GPU per #' encode at DMA speed. Default follows #' \code{options(diffuseR.pin_staging)}. +#' @param stream Logical. With \code{device = "cpu"}, stage the model +#' layer by layer instead of whole: an encode on the GPU then holds +#' only the embeddings and one layer at a time, for cards that cannot +#' hold the ~8 GB NF4 model. #' @param verbose Logical. #' #' @return A \code{gemma4_text_model}. @@ -171,7 +175,7 @@ gemma4_quantize_nf4 <- function(path, output_dir = NULL, shard_bytes = 1.9e9, #' @export load_gemma4_nf4 <- function(artifact_dir, device = "cuda", dtype = "bfloat16", pin = getOption("diffuseR.pin_staging", TRUE), - verbose = TRUE) { + stream = FALSE, verbose = TRUE) { ckpt <- ltx23_open_fp8_checkpoint(artifact_dir) manifest <- jsonlite::fromJSON(file.path(artifact_dir, "manifest.json"), simplifyVector = FALSE) @@ -225,7 +229,13 @@ load_gemma4_nf4 <- function(artifact_dir, device = "cuda", dtype = "bfloat16", model$to(device = device) model$eval() - if (pin && device == "cpu") { + if (stream && device == "cpu") { + attr(model, "staging") <- .stage_layer_stream(model, model$layers, + "layers.", pin) + if (verbose) { + message("Gemma 4 layers staged for streaming") + } + } else if (pin && device == "cpu") { st <- .pin_component(model) if (!is.null(st)) { attr(model, "staging") <- st diff --git a/R/staging.R b/R/staging.R index 8705ad4..8562d82 100644 --- a/R/staging.R +++ b/R/staging.R @@ -171,6 +171,19 @@ NULL invisible(NULL) } +# Layer streaming for a CPU-resident encoder: each layer of `layers` +# (whose parameter names start with `prefix` in the model) gets its own +# staging set on model$layer_stream, which the encoder's forward swaps +# in and out around that layer; the returned pairs cover the rest of the +# model (embeddings, final norm) for the encode-phase onload. +.stage_layer_stream <- function(model, layers, prefix, pin) { + model$layer_stream <- lapply(seq_along(layers), function(i) { + .stage_pairs(c(layers[[i]]$parameters, layers[[i]]$buffers), pin) + }) + tensors <- c(model$parameters, model$buffers) + .stage_pairs(tensors[!startsWith(names(tensors), prefix)], pin) +} + #' Return a pinned component to the CPU #' #' Weights are immutable during inference, so the pinned host copies diff --git a/R/stream_ltx23.R b/R/stream_ltx23.R new file mode 100644 index 0000000..7f4be02 --- /dev/null +++ b/R/stream_ltx23.R @@ -0,0 +1,84 @@ +# LTX on small cards: the transformer keeps a budget of its leading +# blocks on the GPU and streams the rest from (pinned) host memory, one +# block per call, every step. Blocks are ~0.25 GB in NF4 (22B over 48 +# blocks), so streaming k of them costs k * 0.25 GB of DMA per step. + +# Run f() with a streamed unit of `module` (module$unit_stream[[unit]], +# a staging set) on `device`; a plain call when the unit is not streamed +.unit_call <- function(module, unit, device, f) { + st <- module$unit_stream[[unit]] + if (is.null(st)) { + return(f()) + } + .staged_onload(st, device) + on.exit({ + .staged_offload(st) + gc(verbose = FALSE, full = FALSE) + }) + f() +} + +# Stream the text connectors (5.9 GB in bf16): the two input projections +# (1.5 and 0.8 GB) as units, each connector's blocks one at a time. +# Returns the staging pairs for the rest (registers, norms, rope). +.ltx23_connector_stream <- function(connectors, pin) { + units <- c("video_text_proj_in", "audio_text_proj_in") + connectors$unit_stream <- stats::setNames(lapply(units, function(u) { + m <- connectors[[u]] + .stage_pairs(c(m$parameters, m$buffers), pin) + }), units) + for (cn in c("video_connector", "audio_connector")) { + con <- connectors[[cn]] + con$layer_stream <- lapply(seq_along(con$transformer_blocks), function(i) { + b <- con$transformer_blocks[[i]] + .stage_pairs(c(b$parameters, b$buffers), pin) + }) + } + tensors <- c(connectors$parameters, connectors$buffers) + streamed <- startsWith(names(tensors), "video_text_proj_in.") | + startsWith(names(tensors), "audio_text_proj_in.") | + startsWith(names(tensors), "video_connector.transformer_blocks.") | + startsWith(names(tensors), "audio_connector.transformer_blocks.") + .stage_pairs(tensors[!streamed], pin) +} + +# Bytes held by a module's parameters and buffers +.module_bytes <- function(module) { + ts <- c(module$parameters, module$buffers) + sum(vapply(ts, function(t) t$numel() * t$element_size(), 0)) +} + +# Split `transformer` so at most `resident_gb` of it (non-block tensors +# plus its leading blocks) stays with the phase onload. Sets +# transformer$block_stream (per-block staging for the streamed blocks, +# consumed by the forward) and returns the staging pairs for everything +# else, which replace the transformer's whole-module staging set. +.ltx23_block_stream <- function(transformer, resident_gb, pin) { + blocks <- transformer$transformer_blocks + n <- length(blocks) + block_bytes <- vapply(seq_len(n), function(i) .module_bytes(blocks[[i]]), 0) + other_bytes <- .module_bytes(transformer) - sum(block_bytes) + budget <- resident_gb * 1024 ^ 3 - other_bytes + n_resident <- sum(cumsum(block_bytes) <= budget) + + streamed <- if (n_resident < n) seq(n_resident + 1L, n) else integer(0) + stream_staging <- lapply(streamed, function(i) { + .stage_pairs(c(blocks[[i]]$parameters, blocks[[i]]$buffers), pin) + }) + # Everything not in a streamed block, by identity of the tensor names + names_all <- c(names(transformer$parameters), names(transformer$buffers)) + tensors <- c(transformer$parameters, transformer$buffers) + prefix <- paste0("transformer_blocks.", streamed - 1L, ".") + in_stream <- Reduce(`|`, + lapply(prefix, function(p) startsWith(names_all, p)), + FALSE) + transformer$block_stream <- if (length(streamed)) { + list(n_resident = n_resident, staging = stream_staging) + } else { + NULL + } + list(staging = .stage_pairs(tensors[!in_stream], pin), + n_resident = n_resident, n_streamed = length(streamed), + resident_gb = (other_bytes + sum(block_bytes[seq_len(n_resident)])) / 1024 ^ 3, + streamed_gb = sum(block_bytes[streamed]) / 1024 ^ 3) +} diff --git a/R/txt2vid_ltx23.R b/R/txt2vid_ltx23.R index ca3d164..a7f21a5 100644 --- a/R/txt2vid_ltx23.R +++ b/R/txt2vid_ltx23.R @@ -262,6 +262,13 @@ ltx23_unpack_audio_latents <- function(latents, num_mel_bins) { #' @param phase_offload Logical. Load the small components (connectors, #' VAEs, vocoder) to the CPU; the pipeline moves each onto the compute #' device only for its phase. +#' @param resident_gb Numeric. GB of the transformer kept on the GPU +#' during denoising (default \code{Inf}: all of it). With less, its +#' leading blocks stay resident and the rest stream from (pinned) host +#' memory one block at a time every step, for cards that cannot hold +#' the ~12.5 GB NF4 transformer; under 7 GB the text connectors +#' stream too. Needs \code{phase_offload} on CUDA. See +#' \code{\link{ltx23_memory_profile}}. #' @param verbose Logical. #' #' @return A list with the loaded modules and the checkpoint config, @@ -273,7 +280,12 @@ ltx23_load_pipeline <- function(checkpoint_path, device = "cuda", transformer_device = "cpu", components = c("dit", "connectors", "vae", "audio_vae", "vocoder"), pin = TRUE, attn_chunk = NULL, - phase_offload = TRUE, verbose = TRUE) { + phase_offload = TRUE, resident_gb = Inf, + verbose = TRUE) { + if (is.finite(resident_gb) && !(phase_offload && device == "cuda")) { + stop("resident_gb streams transformer blocks to the GPU; it needs ", + "device = \"cuda\" and phase_offload = TRUE", call. = FALSE) + } if (phase_offload) { component_device <- "cpu" } else { @@ -312,7 +324,13 @@ ltx23_load_pipeline <- function(checkpoint_path, device = "cuda", # decode phases measured 86% of wall time in callback-driven # R gc at the 0.20 default reserved rate. Only-if-unset, so an # explicit user option wins. - if (identical(ckpt$format, "nf4")) { + if (is.finite(resident_gb)) { + # Streaming: what stays resident, plus room for activations. + # A 12 GB footprint on a smaller card would push the reserved + # gate past the card, and R would never gc (not even on an + # allocation that fails). + footprint <- resident_gb + 1 + } else if (identical(ckpt$format, "nf4")) { footprint <- 12 } else { footprint <- 8 @@ -409,8 +427,33 @@ ltx23_load_pipeline <- function(checkpoint_path, device = "cuda", pipe$vocoder <- voc } - if (phase_offload && device == "cuda" && - isTRUE(getOption("diffuseR.pin_staging", TRUE))) { + pin_staging <- phase_offload && device == "cuda" && + isTRUE(getOption("diffuseR.pin_staging", TRUE)) + staging <- list() + if (is.finite(resident_gb) && !is.null(pipe$transformer)) { + # The transformer stages only its resident part; the streamed + # blocks carry their own staging sets (staged even unpinned, so + # the phase onload never moves them) + split <- .ltx23_block_stream(pipe$transformer, resident_gb, + pin = pin_staging) + staging$transformer <- split$staging + if (verbose) { + message(sprintf(paste0("Transformer: %d blocks resident (%.1f GB), ", + "%d streamed (%.1f GB/step)"), + split$n_resident, split$resident_gb, + split$n_streamed, split$streamed_gb)) + } + # The 5.9 GB connector phase does not fit beside an 8 GB card's + # activations either: stream it for budgets under 7 GB + if (resident_gb < 7 && !is.null(pipe$connectors)) { + staging$connectors <- .ltx23_connector_stream(pipe$connectors, + pin = pin_staging) + if (verbose) { + message("Text connectors stream block by block") + } + } + } + if (pin_staging) { # Page-lock every phase-offloaded component once so the # per-render CPU<->GPU moves run at full PCIe rate (offload # becomes a pointer swap; see staging.R). Falls back @@ -418,17 +461,17 @@ ltx23_load_pipeline <- function(checkpoint_path, device = "cuda", if (verbose) { message("Pinning host staging buffers...") } - staging <- list() - for (nm in intersect(c("transformer", "connectors", "vae", - "audio_vae", "vocoder"), names(pipe))) { + for (nm in setdiff(intersect(c("transformer", "connectors", "vae", + "audio_vae", "vocoder"), names(pipe)), + names(staging))) { st <- .pin_component(pipe[[nm]]) if (!is.null(st)) { staging[[nm]] <- st } } - if (length(staging)) { - pipe$staging <- staging - } + } + if (length(staging)) { + pipe$staging <- staging } structure(pipe, class = "ltx23_pipeline") @@ -764,7 +807,13 @@ txt2vid_ltx2 <- function(prompt, pipeline, text_encoder = NULL, text_mask <- connector_embeds$attention_mask$to(device = device) } else { onload("connectors") - connectors_device <- pipeline$connectors$video_text_proj_in$weight$device + # A streamed projection sits on the host between calls; the + # streamed connectors always compute on the device + connectors_device <- if (is.null(pipeline$connectors$unit_stream)) { + pipeline$connectors$video_text_proj_in$weight$device + } else { + device + } torch::with_no_grad({ conn <- pipeline$connectors( prompt_embeds$prompt_embeds$to(device = connectors_device, dtype = compute_dtype), diff --git a/inst/tinytest/test_connectors_ltx23.R b/inst/tinytest/test_connectors_ltx23.R index 83b5fca..b47bcbc 100644 --- a/inst/tinytest/test_connectors_ltx23.R +++ b/inst/tinytest/test_connectors_ltx23.R @@ -75,6 +75,30 @@ expect_true( info = sprintf("video3 max abs diff: %.9g", video3_diff) ) +# Streamed (small cards): the two projections as units, each connector's +# blocks one at a time, the rest in the returned pairs. On the CPU the +# staged moves are host-to-host, so the output is identical. +base <- diffuseR:::.ltx23_connector_stream(conn, pin = FALSE) +expect_equal(sort(names(conn$unit_stream)), + c("audio_text_proj_in", "video_text_proj_in")) +expect_equal(length(conn$video_connector$layer_stream), 2L) +expect_equal(length(conn$audio_connector$layer_stream), 2L) +n_staged <- length(base) + + sum(vapply(conn$unit_stream, length, 0L)) + + sum(vapply(conn$video_connector$layer_stream, length, 0L)) + + sum(vapply(conn$audio_connector$layer_stream, length, 0L)) +expect_equal(n_staged, length(c(conn$parameters, conn$buffers))) +torch::with_no_grad({ + res_s <- conn(fx$c_states, fx$c_mask) +}) +expect_true(max_abs_diff(res_s$video_text_embedding, + res$video_text_embedding) == 0) +expect_true(max_abs_diff(res_s$audio_text_embedding, + res$audio_text_embedding) == 0) +conn$unit_stream <- NULL +conn$video_connector$layer_stream <- NULL +conn$audio_connector$layer_stream <- NULL + # Key mapper expect_equal( ltx23_map_connector_key("model.diffusion_model.video_embeddings_connector.transformer_1d_blocks.0.attn1.q_norm.weight"), diff --git a/inst/tinytest/test_dit_ltx23.R b/inst/tinytest/test_dit_ltx23.R index 5e1b3a3..f71fe60 100644 --- a/inst/tinytest/test_dit_ltx23.R +++ b/inst/tinytest/test_dit_ltx23.R @@ -192,6 +192,33 @@ res_stg <- run_model(spatio_temporal_guidance_blocks = c(1L)) expect_true(max_abs_diff(res_stg$sample, fx$m_out_video_stg) < 1e-4) expect_true(max_abs_diff(res_stg$audio_sample, fx$m_out_audio_stg) < 1e-4) +# --- Block streaming (small cards) --------------------------------------------------- + +# One of the two blocks resident, the other streamed: same output (on the +# CPU the staged moves are host-to-host) +blk_gb <- diffuseR:::.module_bytes(model$transformer_blocks[[1]]) / 1024^3 +other_gb <- (diffuseR:::.module_bytes(model) - + 2 * diffuseR:::.module_bytes(model$transformer_blocks[[1]])) / 1024^3 +split <- diffuseR:::.ltx23_block_stream(model, other_gb + 1.5 * blk_gb, + pin = FALSE) +expect_equal(split$n_resident, 1L) +expect_equal(split$n_streamed, 1L) +expect_equal(length(model$block_stream$staging), 1L) +n_block2 <- length(c(model$transformer_blocks[[2]]$parameters, + model$transformer_blocks[[2]]$buffers)) +expect_equal(length(split$staging), + length(c(model$parameters, model$buffers)) - n_block2) +res_stream <- run_model( + encoder_attention_mask = fx$m_enc_mask, + audio_encoder_attention_mask = fx$m_enc_mask +) +expect_true(max_abs_diff(res_stream$sample, res$sample) == 0) +expect_true(max_abs_diff(res_stream$audio_sample, res$audio_sample) == 0) +# A budget that holds everything streams nothing +split_all <- diffuseR:::.ltx23_block_stream(model, Inf, pin = FALSE) +expect_equal(split_all$n_streamed, 0L) +expect_true(is.null(model$block_stream)) + # --- Key mapper ------------------------------------------------------------------ expect_equal( diff --git a/inst/tinytest/test_gemma4_nf4.R b/inst/tinytest/test_gemma4_nf4.R index aa1357b..22688f5 100644 --- a/inst/tinytest/test_gemma4_nf4.R +++ b/inst/tinytest/test_gemma4_nf4.R @@ -131,6 +131,19 @@ art_seen <- local({ }) expect_true(as.numeric((h_nf4[valid] - h_twin[valid])$abs()$max()) < 1e-5) + # Layer streaming (small cards): the staging attribute holds the + # non-layer tensors, one set per layer on the module; on the CPU the + # staged moves are host-to-host, so the output is identical + enc_s <- load_gemma4_nf4(art, device = "cpu", dtype = "float32", + pin = FALSE, stream = TRUE, verbose = FALSE) + expect_equal(length(enc_s$layer_stream), length(enc_s$layers)) + tensors <- c(enc_s$parameters, enc_s$buffers) + expect_equal(length(attr(enc_s, "staging")), + sum(!startsWith(names(tensors), "layers."))) + h_s <- torch::with_no_grad( + enc_s(fx$input_ids, attention_mask = fx$attention_mask)$last_hidden_state) + expect_true(as.numeric((h_s - h_nf4)$abs()$max()) == 0) + # Default bf16 compute through the encode step stays finite enc_bf <- load_gemma4_nf4(art, device = "cpu", pin = FALSE, verbose = FALSE) e <- encode_with_gemma4(c("the cat sat", "tell"), enc_bf, tok_art, diff --git a/inst/tinytest/test_memory_ltx23.R b/inst/tinytest/test_memory_ltx23.R index 204cc0a..cad3821 100644 --- a/inst/tinytest/test_memory_ltx23.R +++ b/inst/tinytest/test_memory_ltx23.R @@ -47,3 +47,26 @@ clear() expect_equal(ltx23_tune_gc(footprint_gb = 12, total_gb = 12), 0.92) options(old) + +# Profiles: small cards stream NF4 transformer blocks (vram - 4 GB +# resident) and, below 12 GB, the Gemma 4 layers +p16 <- ltx23_memory_profile(16) +expect_equal(p16$name, "high") +expect_equal(p16$resident_gb, Inf) +expect_false(p16$stream_text) +p12 <- ltx23_memory_profile(12) +expect_equal(p12$name, "medium") +expect_equal(p12$precision, "nf4") +expect_equal(p12$resident_gb, 8) +expect_false(p12$stream_text) +p8 <- ltx23_memory_profile(8) +expect_equal(p8$name, "low") +expect_equal(p8$precision, "nf4") +expect_equal(p8$resident_gb, 4) +expect_true(p8$stream_text) +expect_true(ltx23_memory_profile(11)$stream_text) +expect_equal(ltx23_memory_profile(4)$name, "cpu_only") + +# The loader refuses block streaming it cannot do +expect_error(ltx23_load_pipeline("unused", device = "cpu", resident_gb = 4), + "phase_offload") diff --git a/man/load_gemma4_nf4.Rd b/man/load_gemma4_nf4.Rd index abb90fa..bfdc023 100644 --- a/man/load_gemma4_nf4.Rd +++ b/man/load_gemma4_nf4.Rd @@ -8,6 +8,7 @@ load_gemma4_nf4( device = "cuda", dtype = "bfloat16", pin = getOption("diffuseR.pin_staging", TRUE), + stream = FALSE, verbose = TRUE ) } @@ -23,6 +24,11 @@ load_gemma4_nf4( encode at DMA speed. Default follows \code{options(diffuseR.pin_staging)}.} +\item{stream}{Logical. With \code{device = "cpu"}, stage the model +layer by layer instead of whole: an encode on the GPU then holds +only the embeddings and one layer at a time, for cards that cannot +hold the ~8 GB NF4 model.} + \item{verbose}{Logical.} } \value{ diff --git a/man/load_gemma4_text_encoder.Rd b/man/load_gemma4_text_encoder.Rd index 2389aa2..3b320e4 100644 --- a/man/load_gemma4_text_encoder.Rd +++ b/man/load_gemma4_text_encoder.Rd @@ -8,6 +8,7 @@ load_gemma4_text_encoder( device = "cpu", dtype = "bfloat16", pin = getOption("diffuseR.pin_staging", TRUE), + stream = FALSE, verbose = TRUE ) } @@ -27,6 +28,9 @@ directory from \code{\link{gemma4_quantize_nf4}} (dispatched to encode at DMA speed. Default follows \code{options(diffuseR.pin_staging)}.} +\item{stream}{Logical. Stage layer by layer (see +\code{\link{load_gemma4_nf4}}).} + \item{verbose}{Logical.} } \value{ diff --git a/man/ltx23_load_pipeline.Rd b/man/ltx23_load_pipeline.Rd index c982fdb..5bde5f5 100644 --- a/man/ltx23_load_pipeline.Rd +++ b/man/ltx23_load_pipeline.Rd @@ -12,6 +12,7 @@ ltx23_load_pipeline( pin = TRUE, attn_chunk = NULL, phase_offload = TRUE, + resident_gb = Inf, verbose = TRUE ) } @@ -43,6 +44,14 @@ weights when loading the plain (non-fp8) checkpoint.} VAEs, vocoder) to the CPU; the pipeline moves each onto the compute device only for its phase.} +\item{resident_gb}{Numeric. GB of the transformer kept on the GPU +during denoising (default \code{Inf}: all of it). With less, its +leading blocks stay resident and the rest stream from (pinned) host +memory one block at a time every step, for cards that cannot hold +the ~12.5 GB NF4 transformer; under 7 GB the text connectors +stream too. Needs \code{phase_offload} on CUDA. See +\code{\link{ltx23_memory_profile}}.} + \item{verbose}{Logical.} } \value{ diff --git a/man/ltx23_memory_profile.Rd b/man/ltx23_memory_profile.Rd index cd2b2f8..bf7c26f 100644 --- a/man/ltx23_memory_profile.Rd +++ b/man/ltx23_memory_profile.Rd @@ -10,7 +10,8 @@ ltx23_memory_profile(vram_gb = NULL) } \value{ Named list with device/dtype placement, \code{attn_chunk}, - \code{pin_weights}, and resolution caps. + \code{pin_weights}, \code{resident_gb}, \code{stream_text}, and + resolution caps. } \description{ Selects transformer precision, component placement, and attention @@ -23,9 +24,18 @@ renders 1280x704x121 with audio in ~23 min at a 15.7 GB peak \code{diffuseR.attn_budget} of 1.5e8 all required at that size). } \details{ +Below 14 GB the NF4 transformer keeps \code{resident_gb} on the GPU +and streams its remaining blocks from pinned host memory every step +(pass it to \code{\link{ltx23_load_pipeline}}), and below 12 GB the +Gemma 4 encoder streams its layers (\code{stream_text}; pass it as +\code{stream} to \code{\link{load_gemma4_text_encoder}}). Measured +with the rest of a 16 GB card held back: LTX-2.5 768x512x121 renders +in ~84 s on both 12 GB (8 GB resident) and 8 GB (4 GB resident), the +same as on the full card. + \describe{ -\item{precision "nf4"}{Weights resident on the GPU; fastest steps; -about 9 percent weight round-trip error.} +\item{precision "nf4"}{Weights resident on the GPU (or partly +streamed); fastest steps; about 9 percent weight round-trip error.} \item{precision "fp8"}{Weights CPU-resident, streamed per linear; near-bf16 quality; each step pays the PCIe transfer.} } From fc719de4623e73d278289c68e15994ee1a5fb641 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Wed, 28 Oct 2026 17:33:17 -0500 Subject: [PATCH 2/3] Bump version to 0.2.2.18 --- DESCRIPTION | 2 +- NEWS.md | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 04adcfd..465b631 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: diffuseR Title: Functional Interface to Diffusion Models in R -Version: 0.2.2.17 +Version: 0.2.2.18 Authors@R: c( person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"), comment = c(ORCID = "0009-0005-4248-604X")), diff --git a/NEWS.md b/NEWS.md index 899aa18..45d716f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,27 @@ +# diffuseR 0.2.2.18 + +* **LTX-2.5 on 8 and 12 GB cards.** The NF4 transformer can keep part + of itself on the GPU and stream the rest from pinned host memory, one + block per call, every step: + - `ltx23_load_pipeline(resident_gb = )` sets how much stays resident + (default `Inf`, all of it). Under 7 GB the text connectors (5.9 GB) + stream too: their two projections as units, their blocks one at a + time. + - `load_gemma4_nf4(stream = TRUE)` / `load_gemma4_text_encoder()` + stage Gemma 4 layer by layer, so an encode holds the embeddings and + one layer at a time (3 GB peak instead of 8.8 GB). + - `ltx23_memory_profile()` now returns `resident_gb` and + `stream_text`; its 10-14 GB and 7-10 GB tiers use NF4 with + streaming instead of fp8, which LTX-2.5 has no artifact for. + - Measured with the rest of a 16 GB card held back, 768x512x121: + ~84 s on 12 GB (8 GB resident, pipeline peak 9.6 GB) and on 8 GB + (4 GB resident, peak 5.7 GB), the same as the full card. +* Streaming keeps the output: once TorchScript has warmed up, streamed + and resident renders are bit-identical. The first render in a process + still differs from later ones with the same seed (the profiling + executor switches graphs after its first calls); that predates this + release. + # diffuseR 0.2.2.17 * **FLUX 3 on 8 and 12 GB cards.** `flux3_load_pipeline()` plans from From affa7c7537f0f23ef63d4697c49bfde219346b9d Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Wed, 28 Oct 2026 20:59:51 -0500 Subject: [PATCH 3/3] LTX with less host RAM: split text phase and disk-backed resident blocks ltx23_text_embeds() runs the connectors on a connectors-only pipeline (optionally from raw prompt_embeds), so Gemma, the connectors and the render pipeline never share host RAM. ltx23_load_pipeline(resident_from = "disk") keeps no host copy of the GPU-resident NF4 blocks and re-reads them from the artifact per render. pin = FALSE now also leaves staging buffers unpinned. ltx23_memory_profile() picks split_text and resident_from from available host RAM. --- CLAUDE.md | 15 ++- NAMESPACE | 1 + NEWS.md | 22 ++++ R/memory_ltx23.R | 43 ++++++- R/nf4_ltx23.R | 8 ++ R/staging.R | 62 +++++++++- R/stream_ltx23.R | 20 +++- R/txt2vid_ltx23.R | 193 +++++++++++++++++++++++++----- inst/tinytest/test_dit_ltx23.R | 33 +++++ inst/tinytest/test_memory_ltx23.R | 27 +++++ man/ltx23_load_pipeline.Rd | 14 ++- man/ltx23_memory_profile.Rd | 21 +++- man/ltx23_text_embeds.Rd | 70 +++++++++++ 13 files changed, 482 insertions(+), 47 deletions(-) create mode 100644 man/ltx23_text_embeds.Rd diff --git a/CLAUDE.md b/CLAUDE.md index fe17834..ac869d9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -509,8 +509,19 @@ Gotcha that cost an hour: the first render in a process differs from later ones with the same seed (TorchScript's profiling executor swaps graphs after its first calls; max latent diff ~3), so compare streamed vs resident only after a warm-up run. Host RAM: the pipeline's pinned -staging (~21 GB) plus Gemma 4 do not fit this host's free RAM in one -process; set `MALLOC_MMAP_THRESHOLD_=1048576` and encode separately. +staging plus Gemma 4 need ~31 GB in one process. Low-RAM flow: encode +with Gemma, free it, `ltx23_text_embeds()` on a connectors-only +pipeline loaded `pin = FALSE`, free it, then load the render pipeline +without connectors, optionally `resident_from = "disk"` (no host copy +of the resident blocks; re-read at ~1 GB/s). Measured high-water marks +at 768x512x121: text 12.4 GB; render 20.1 GB (host) / 16.0 GB (disk) +on 8 GB, 14.9 GB (disk) on 12 GB. Set `MALLOC_MMAP_THRESHOLD_=1048576` +so freed tensors return to the OS. + +Gotchas that bit once (host RAM): freed PINNED memory stays in torch's +CachingHostAllocator, not the OS, so a pipeline you plan to drop must +load unpinned (RSS stayed 9 GB after freeing pinned connectors). Pinned +pages show up as RssShmem in /proc/self/status, not RssAnon. ### LTX-2.3 Video Generation (clean-room rewrite in progress) diff --git a/NAMESPACE b/NAMESPACE index de07e6d..8fb40fb 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -182,6 +182,7 @@ export(ltx23_split_keys) export(ltx23_stage2_distilled_sigmas) export(ltx23_tail_latents) export(ltx23_text_connectors) +export(ltx23_text_embeds) export(ltx23_tone_map_latents) export(ltx23_transformer) export(ltx23_transformer_block) diff --git a/NEWS.md b/NEWS.md index 45d716f..0e1e362 100644 --- a/NEWS.md +++ b/NEWS.md @@ -16,6 +16,28 @@ - Measured with the rest of a 16 GB card held back, 768x512x121: ~84 s on 12 GB (8 GB resident, pipeline peak 9.6 GB) and on 8 GB (4 GB resident, peak 5.7 GB), the same as the full card. +* **LTX with less host RAM.** The full small-card flow in one process + needs about 31 GB (the pinned render pipeline plus Gemma 4). Two ways + to need less: + - `ltx23_text_embeds()` runs the connectors on a connectors-only + pipeline and returns the embeddings for `txt2vid_ltx2(connector_embeds = )`. + It takes raw `prompt_embeds`, so you encode with Gemma, free it, run + the connectors, free them, and only then load the render pipeline + (`components` without `"connectors"`). The text phase peaks at + 12.4 GB. + - `ltx23_load_pipeline(resident_from = "disk")` keeps no host copy of + the GPU-resident NF4 blocks and re-reads them from the artifact each + render (~1 GB/s through safetensors). + - `ltx23_load_pipeline(pin = FALSE)` now also leaves the staging + buffers unpinned. Freed pinned memory stays in torch's host cache, + so a pipeline you will drop should load unpinned. + - `ltx23_memory_profile(host_ram_gb = )` (detected by default) returns + `split_text` (below 32 GB) and `resident_from` (`"disk"` below + 22 GB). + - Measured at 768x512x121 under a 20 GB cgroup cap, split text phase, + second render. 8 GB card: host copies 20.1 GB high-water, 82 s; + disk 16.0 GB, 90 s. 12 GB card, disk: 14.9 GB (the load), 10.4 GB + while rendering, 92 s. Host and disk renders are bit-identical. * Streaming keeps the output: once TorchScript has warmed up, streamed and resident renders are bit-identical. The first render in a process still differs from later ones with the same seed (the profiling diff --git a/R/memory_ltx23.R b/R/memory_ltx23.R index a9ff152..f8178eb 100644 --- a/R/memory_ltx23.R +++ b/R/memory_ltx23.R @@ -36,18 +36,36 @@ NULL #' near-bf16 quality; each step pays the PCIe transfer.} #' } #' +#' Host RAM decides two more fields for the GPU tiers. \code{split_text} +#' asks the caller to encode in its own phase (Gemma, then the +#' connectors, each freed before the next loads; see +#' \code{\link{ltx23_text_embeds}}) instead of holding the encoder next +#' to the render pipeline. \code{resident_from} is passed to +#' \code{\link{ltx23_load_pipeline}}: \code{"disk"} drops the host copy +#' of the GPU-resident transformer blocks. Below 32 GB available the +#' text phase splits off; below 22 GB the resident blocks come from +#' disk (measured on LTX-2.5 at 768x512x121 with an 8 GB card: the +#' render phase peaks at 20.1 GB of host RAM with host copies and 16.0 +#' GB without, for ~8 s more per render). +#' #' @param vram_gb Numeric or NULL (auto-detect free VRAM). +#' @param host_ram_gb Numeric or NULL (auto-detect available RAM; NA +#' when undetectable, which keeps the all-in-RAM defaults). #' #' @return Named list with device/dtype placement, \code{attn_chunk}, -#' \code{pin_weights}, \code{resident_gb}, \code{stream_text}, and -#' resolution caps. +#' \code{pin_weights}, \code{resident_gb}, \code{stream_text}, +#' \code{split_text}, \code{resident_from}, and resolution caps. #' #' @export -ltx23_memory_profile <- function(vram_gb = NULL) { +ltx23_memory_profile <- function(vram_gb = NULL, host_ram_gb = NULL) { if (is.null(vram_gb)) { vram_gb <- .detect_vram(use_free = TRUE) message(sprintf("Detected %.1f GB free VRAM", vram_gb)) } + if (is.null(host_ram_gb)) { + host_ram_gb <- .detect_host_ram() + } + ram_known <- isTRUE(is.finite(host_ram_gb)) profile <- if (vram_gb >= 14) { "high" @@ -114,9 +132,26 @@ ltx23_memory_profile <- function(vram_gb = NULL) { ) ) - profiles[[profile]] + out <- profiles[[profile]] + gpu <- out$device == "cuda" + out$split_text <- gpu && ram_known && host_ram_gb < .ltx23_ram_one_process + out$resident_from <- if (gpu && ram_known && + host_ram_gb < .ltx23_ram_host_resident) { + "disk" + } else { + "host" + } + out } +# Host RAM (GB) for the LTX GPU tiers: the render pipeline plus the +# Gemma 4 encoder in one process, and the render pipeline alone with +# its resident blocks kept in host memory (LTX-2.5 768x512x121 on a +# simulated 8 GB card, high-water marks: 20.1 GB with host copies, +# 16.0 GB re-reading from disk, 12.4 GB for the split text phase) +.ltx23_ram_one_process <- 32 +.ltx23_ram_host_resident <- 22 + #' Tune the torch CUDA allocator for large-resident inference #' #' Stops the allocator GC storm (see the "Performance Levers" vignette diff --git a/R/nf4_ltx23.R b/R/nf4_ltx23.R index 0ba3a36..e717e81 100644 --- a/R/nf4_ltx23.R +++ b/R/nf4_ltx23.R @@ -416,6 +416,9 @@ ltx23_load_transformer_nf4 <- function(ckpt, device = "cuda", verbose = TRUE, dests <- c(model$named_parameters(), model$named_buffers()) filled <- character(0) unmapped <- character(0) + # Model tensor name -> artifact key, so a staging set can re-read a + # tensor from disk instead of keeping a host copy (.stage_from_disk) + source_keys <- character(0) torch::with_no_grad({ for (i in seq_along(main_keys)) { @@ -424,6 +427,9 @@ ltx23_load_transformer_nf4 <- function(ckpt, device = "cuda", verbose = TRUE, if (ltx23_is_fp8_cast_key(mapped) && paste0(key, "_absmax") %in% absmax_keys) { + stem <- sub("\\.weight$", "", mapped) + source_keys[paste0(stem, c(".weight_nf4", ".weight_absmax"))] <- + c(key, paste0(key, "_absmax")) segments <- strsplit(mapped, ".", fixed = TRUE)[[1]] parent <- .ltx23_walk_module(model, utils::head(segments, -2L)) leaf <- segments[length(segments) - 1L] @@ -454,6 +460,7 @@ ltx23_load_transformer_nf4 <- function(ckpt, device = "cuda", verbose = TRUE, } dest$copy_(ckpt$handle$get_tensor(key)) filled <- c(filled, mapped) + source_keys[mapped] <- key } if (i %% 100L == 0L) { @@ -481,6 +488,7 @@ ltx23_load_transformer_nf4 <- function(ckpt, device = "cuda", verbose = TRUE, # Everything (packed weights included, as buffers) onto the GPU model$to(device = device) model$eval() + model$weight_source <- list(keys = source_keys, handle = ckpt$handle) # Block intermediates (norms, projections, FF activations) are still # ~1.5GB per block at high resolution; per-block gc keeps them bounded options(diffuseR.block_gc = TRUE) diff --git a/R/staging.R b/R/staging.R index 8562d82..9a489b1 100644 --- a/R/staging.R +++ b/R/staging.R @@ -162,15 +162,68 @@ NULL #' @keywords internal .staged_onload <- function(staging, device) { spec <- .device_spec(device) + n_read <- 0L for (pair in staging) { - if (.on_device(pair$live, spec)) { + disk <- is.null(pair$pinned) + if (.on_device(pair$live, spec) && (!disk || pair$state$loaded)) { + next + } + if (disk) { + # Disk-backed pair (.stage_from_disk): read it again. The host + # copies are garbage once on the device; collect them as we + # go so the read never holds more than a slice in RAM + pair$live$set_data(pair$read()$to(device = device, + dtype = pair$live$dtype)) + pair$state$loaded <- TRUE + n_read <- n_read + 1L + if (n_read %% 64L == 0L) { + gc(verbose = FALSE, full = FALSE) + } next } pair$live$set_data(pair$pinned$to(device = device, non_blocking = TRUE)) } + if (n_read) { + gc(verbose = FALSE, full = FALSE) + } invisible(NULL) } +# Staging pairs that keep NO host copy: `keys` names each tensor's +# artifact key (a model's weight_source$keys) and `handle` reads it. +# Offload drops the device copy and onload re-reads the tensor from +# disk (the OS page cache, when RAM allows). Tensors without a key +# (computed buffers) get ordinary pairs. Releases the host copies now. +.stage_from_disk <- function(tensors, keys, handle, pin) { + on_disk <- names(tensors) %in% names(keys) + pairs <- vector("list", length(tensors)) + names(pairs) <- names(tensors) + pairs[!on_disk] <- .stage_pairs(tensors[!on_disk], pin) + for (nm in names(tensors)[on_disk]) { + key <- keys[[nm]] + state <- new.env(parent = emptyenv()) + state$loaded <- FALSE + pair <- list(live = tensors[[nm]], state = state, + read = local({ + k <- key + function() handle$get_tensor(k) + })) + .release_host(pair$live) + pairs[[nm]] <- pair + } + gc(verbose = FALSE) + pairs +} + +# Point a tensor at a zero-stride placeholder of the same shape and +# dtype: its storage is a single element, so the old data is freed +.release_host <- function(t) { + shape <- as.integer(t$shape) + stub <- torch::torch_zeros(1L, dtype = t$dtype) + stub <- if (length(shape)) stub$expand(shape) else stub$squeeze() + t$set_data(stub) +} + # Layer streaming for a CPU-resident encoder: each layer of `layers` # (whose parameter names start with `prefix` in the model) gets its own # staging set on model$layer_stream, which the encoder's forward swaps @@ -192,7 +245,12 @@ NULL #' @keywords internal .staged_offload <- function(staging) { for (pair in staging) { - pair$live$set_data(pair$pinned) + if (is.null(pair$pinned)) { + .release_host(pair$live) + pair$state$loaded <- FALSE + } else { + pair$live$set_data(pair$pinned) + } } invisible(NULL) } diff --git a/R/stream_ltx23.R b/R/stream_ltx23.R index 7f4be02..9375e62 100644 --- a/R/stream_ltx23.R +++ b/R/stream_ltx23.R @@ -52,8 +52,11 @@ # plus its leading blocks) stays with the phase onload. Sets # transformer$block_stream (per-block staging for the streamed blocks, # consumed by the forward) and returns the staging pairs for everything -# else, which replace the transformer's whole-module staging set. -.ltx23_block_stream <- function(transformer, resident_gb, pin) { +# else, which replace the transformer's whole-module staging set. With +# `from_disk`, that resident part keeps no host copy: it is re-read from +# the artifact at each onload (.stage_from_disk). +.ltx23_block_stream <- function(transformer, resident_gb, pin, + from_disk = FALSE) { blocks <- transformer$transformer_blocks n <- length(blocks) block_bytes <- vapply(seq_len(n), function(i) .module_bytes(blocks[[i]]), 0) @@ -77,7 +80,18 @@ } else { NULL } - list(staging = .stage_pairs(tensors[!in_stream], pin), + staging <- if (from_disk) { + src <- transformer$weight_source + if (is.null(src)) { + stop("resident_from = \"disk\" needs an NF4 transformer ", + "(ltx23_load_transformer_nf4 records its artifact keys)", + call. = FALSE) + } + .stage_from_disk(tensors[!in_stream], src$keys, src$handle, pin) + } else { + .stage_pairs(tensors[!in_stream], pin) + } + list(staging = staging, n_resident = n_resident, n_streamed = length(streamed), resident_gb = (other_bytes + sum(block_bytes[seq_len(n_resident)])) / 1024 ^ 3, streamed_gb = sum(block_bytes[streamed]) / 1024 ^ 3) diff --git a/R/txt2vid_ltx23.R b/R/txt2vid_ltx23.R index a7f21a5..55a2671 100644 --- a/R/txt2vid_ltx23.R +++ b/R/txt2vid_ltx23.R @@ -256,7 +256,10 @@ ltx23_unpack_audio_latents <- function(latents, num_mel_bins) { #' @param transformer_device Character. Device for the transformer #' weights when loading the plain (non-fp8) checkpoint. #' @param components Character vector. Which components to load. -#' @param pin Logical. Pin fp8 host memory (fp8 artifact only). +#' @param pin Logical. Pin host memory: the fp8 weights, and the +#' phase-offload staging buffers (with \code{options(diffuseR.pin_staging)}). +#' Freed pinned memory stays in torch's host cache, so pass +#' \code{FALSE} for a pipeline that will be dropped to make room. #' @param attn_chunk Integer or NULL. Query-chunk size for attention #' (see \code{\link{ltx23_set_attn_chunk}}). #' @param phase_offload Logical. Load the small components (connectors, @@ -269,6 +272,13 @@ ltx23_unpack_audio_latents <- function(latents, num_mel_bins) { #' the ~12.5 GB NF4 transformer; under 7 GB the text connectors #' stream too. Needs \code{phase_offload} on CUDA. See #' \code{\link{ltx23_memory_profile}}. +#' @param resident_from Where the GPU-resident part of an NF4 +#' transformer comes back from after each render's decode phase: +#' \code{"host"} (default) keeps a pinned host copy, \code{"disk"} +#' keeps none and re-reads it from the artifact at the next onload +#' (the OS page cache serves it when RAM allows). \code{"disk"} saves +#' host RAM equal to the resident part (up to ~12 GB) for a re-read +#' per render. Needs \code{phase_offload} on CUDA. #' @param verbose Logical. #' #' @return A list with the loaded modules and the checkpoint config, @@ -281,10 +291,15 @@ ltx23_load_pipeline <- function(checkpoint_path, device = "cuda", components = c("dit", "connectors", "vae", "audio_vae", "vocoder"), pin = TRUE, attn_chunk = NULL, phase_offload = TRUE, resident_gb = Inf, + resident_from = c("host", "disk"), verbose = TRUE) { - if (is.finite(resident_gb) && !(phase_offload && device == "cuda")) { - stop("resident_gb streams transformer blocks to the GPU; it needs ", - "device = \"cuda\" and phase_offload = TRUE", call. = FALSE) + resident_from <- match.arg(resident_from) + from_disk <- resident_from == "disk" + if ((is.finite(resident_gb) || from_disk) && + !(phase_offload && device == "cuda")) { + stop("resident_gb and resident_from = \"disk\" stage the transformer ", + "through the GPU; they need device = \"cuda\" and ", + "phase_offload = TRUE", call. = FALSE) } if (phase_offload) { component_device <- "cpu" @@ -427,30 +442,34 @@ ltx23_load_pipeline <- function(checkpoint_path, device = "cuda", pipe$vocoder <- voc } - pin_staging <- phase_offload && device == "cuda" && + # Pinned pages go to torch's host cache when freed, not back to the + # OS: a pipeline that will be dropped (the text phase) loads unpinned + pin_staging <- phase_offload && device == "cuda" && isTRUE(pin) && isTRUE(getOption("diffuseR.pin_staging", TRUE)) staging <- list() - if (is.finite(resident_gb) && !is.null(pipe$transformer)) { + if ((is.finite(resident_gb) || from_disk) && !is.null(pipe$transformer)) { # The transformer stages only its resident part; the streamed # blocks carry their own staging sets (staged even unpinned, so # the phase onload never moves them) split <- .ltx23_block_stream(pipe$transformer, resident_gb, - pin = pin_staging) + pin = pin_staging, from_disk = from_disk) staging$transformer <- split$staging if (verbose) { - message(sprintf(paste0("Transformer: %d blocks resident (%.1f GB), ", + message(sprintf(paste0("Transformer: %d blocks resident (%.1f GB%s), ", "%d streamed (%.1f GB/step)"), split$n_resident, split$resident_gb, + if (from_disk) ", re-read from disk" else "", split$n_streamed, split$streamed_gb)) } - # The 5.9 GB connector phase does not fit beside an 8 GB card's - # activations either: stream it for budgets under 7 GB - if (resident_gb < 7 && !is.null(pipe$connectors)) { - staging$connectors <- .ltx23_connector_stream(pipe$connectors, - pin = pin_staging) - if (verbose) { - message("Text connectors stream block by block") - } + } + # The 5.9 GB connector phase does not fit beside an 8 GB card's + # activations either: stream it for budgets under 7 GB (also when + # the connectors load alone, for ltx23_text_embeds) + if (resident_gb < 7 && !is.null(pipe$connectors)) { + staging$connectors <- .ltx23_connector_stream(pipe$connectors, + pin = pin_staging) + if (verbose) { + message("Text connectors stream block by block") } } if (pin_staging) { @@ -477,6 +496,127 @@ ltx23_load_pipeline <- function(checkpoint_path, device = "cuda", structure(pipe, class = "ltx23_pipeline") } +# Run the (onloaded) text connectors on raw encoder states; returns the +# connector outputs on `device`. A streamed projection sits on the host +# between calls, so streamed connectors always compute on the device. +.ltx23_run_connectors <- function(connectors, prompt_embeds, device, + compute_dtype) { + conn_device <- if (is.null(connectors$unit_stream)) { + connectors$video_text_proj_in$weight$device + } else { + device + } + torch::with_no_grad({ + conn <- connectors( + prompt_embeds$prompt_embeds$to(device = conn_device, + dtype = compute_dtype), + prompt_embeds$prompt_attention_mask$to(device = conn_device) + ) + list(video_text_embedding = conn$video_text_embedding$to(device = device), + audio_text_embedding = conn$audio_text_embedding$to(device = device), + attention_mask = conn$attention_mask$to(device = device)) + }) +} + +#' Encode an LTX prompt to connector embeddings +#' +#' Runs the text encoder and the text connectors and returns their +#' output on the CPU, ready for the \code{connector_embeds} argument of +#' \code{\link{txt2vid_ltx2}}. This splits a render into phases that +#' never share host RAM: encode with Gemma and drop it, run a +#' connectors-only pipeline (loaded with \code{pin = FALSE}) and drop +#' it, then load the render pipeline without connectors. +#' +#' @param prompt Character. The prompt. +#' @param pipeline An \code{ltx23_pipeline} holding the connectors, e.g. +#' \code{ltx23_load_pipeline(path, components = "connectors")}. +#' @param text_encoder,tokenizer The Gemma 3 or Gemma 4 model and +#' tokenizer (see \code{\link{txt2vid_ltx2}}). Ignored when +#' \code{prompt_embeds} is supplied. +#' @param prompt_embeds Optional raw encoder output (the result of +#' \code{\link{encode_with_gemma4}} or \code{\link{encode_with_gemma3}}), +#' so the encoder can be freed before the connectors load. +#' @param device Character. Compute device. +#' @param dtype Character. Compute dtype. +#' @param max_sequence_length Integer. Text token length. +#' @param verbose Logical. +#' +#' @return A list with \code{video_text_embedding}, +#' \code{audio_text_embedding} and \code{attention_mask}, on the CPU. +#' +#' @examples +#' \dontrun{ +#' ckpt <- file.path(tools::R_user_dir("diffuseR", "data"), "ltx2.5-nf4-full") +#' gemma <- file.path(tools::R_user_dir("diffuseR", "data"), "gemma4-nf4") +#' prompt <- "A red fox in fresh snow" +#' te <- load_gemma4_nf4(gemma, device = "cpu", stream = TRUE, pin = FALSE) +#' raw <- encode_with_gemma4(prompt, te, gemma4_tokenizer(gemma)) +#' rm(te); gc() +#' text <- ltx23_load_pipeline(ckpt, components = "connectors", +#' resident_gb = 4, pin = FALSE) +#' emb <- ltx23_text_embeds(prompt, text, prompt_embeds = raw) +#' rm(text, raw); gc() +#' pipe <- ltx23_load_pipeline(ckpt, components = c("dit", "vae", +#' "audio_vae", "vocoder"), resident_gb = 4, +#' resident_from = "disk") +#' txt2vid_ltx2(prompt, pipe, connector_embeds = emb, filename = "fox.mp4") +#' } +#' +#' @export +ltx23_text_embeds <- function(prompt, pipeline, text_encoder = NULL, + tokenizer = NULL, prompt_embeds = NULL, + device = "cuda", dtype = "bfloat16", + max_sequence_length = 1024L, verbose = TRUE) { + stopifnot(inherits(pipeline, "ltx23_pipeline")) + if (is.null(pipeline$connectors)) { + stop("pipeline has no connectors; load it with components ", + "including \"connectors\"", call. = FALSE) + } + compute_dtype <- switch(dtype, bfloat16 = torch::torch_bfloat16(), + float16 = torch::torch_float16(), + float32 = torch::torch_float32(), + stop("Unsupported dtype: ", dtype)) + raw <- prompt_embeds + if (is.null(raw)) { + if (is.null(text_encoder) || is.null(tokenizer)) { + stop("Supply text_encoder + tokenizer, or prompt_embeds.", + call. = FALSE) + } + encode <- if (inherits(tokenizer, "gemma4_tokenizer")) { + encode_with_gemma4 + } else { + encode_with_gemma3 + } + raw <- encode(prompt, model = text_encoder, tokenizer = tokenizer, + max_sequence_length = max_sequence_length, + device = .ltx23_text_encode_device(text_encoder, device), + verbose = verbose) + } + st <- pipeline$staging$connectors + on_gpu <- device != "cpu" + if (on_gpu) { + if (is.null(st)) { + pipeline$connectors$to(device = device) + } else { + .staged_onload(st, device) + } + } + conn <- .ltx23_run_connectors(pipeline$connectors, raw, device, + compute_dtype) + rm(raw) + out <- lapply(conn, function(t) t$to(device = "cpu")) + rm(conn) + if (on_gpu) { + if (is.null(st)) { + pipeline$connectors$to(device = "cpu") + } else { + .staged_offload(st) + } + } + gc(verbose = FALSE) + out +} + # The device the prompt encode runs on, for the text_encoder txt2vid_ltx2 # was handed. A PATH is loaded fresh onto `device` -- a caller who names a # file is asking for it there. A PRELOADED object is CPU-resident by the @@ -807,22 +947,11 @@ txt2vid_ltx2 <- function(prompt, pipeline, text_encoder = NULL, text_mask <- connector_embeds$attention_mask$to(device = device) } else { onload("connectors") - # A streamed projection sits on the host between calls; the - # streamed connectors always compute on the device - connectors_device <- if (is.null(pipeline$connectors$unit_stream)) { - pipeline$connectors$video_text_proj_in$weight$device - } else { - device - } - torch::with_no_grad({ - conn <- pipeline$connectors( - prompt_embeds$prompt_embeds$to(device = connectors_device, dtype = compute_dtype), - prompt_embeds$prompt_attention_mask$to(device = connectors_device) - ) - video_text_embeds <- conn$video_text_embedding$to(device = device) - audio_text_embeds <- conn$audio_text_embedding$to(device = device) - text_mask <- conn$attention_mask$to(device = device) - }) + conn <- .ltx23_run_connectors(pipeline$connectors, prompt_embeds, + device, compute_dtype) + video_text_embeds <- conn$video_text_embedding + audio_text_embeds <- conn$audio_text_embedding + text_mask <- conn$attention_mask rm(conn) offload("connectors") } diff --git a/inst/tinytest/test_dit_ltx23.R b/inst/tinytest/test_dit_ltx23.R index f71fe60..52ed496 100644 --- a/inst/tinytest/test_dit_ltx23.R +++ b/inst/tinytest/test_dit_ltx23.R @@ -219,6 +219,39 @@ split_all <- diffuseR:::.ltx23_block_stream(model, Inf, pin = FALSE) expect_equal(split_all$n_streamed, 0L) expect_true(is.null(model$block_stream)) +# Resident part re-read from disk (resident_from = "disk"): staging drops +# the host copies, the onload reads them back, same output +if (requireNamespace("safetensors", quietly = TRUE)) { + tensors <- c(model$parameters, model$buffers) + st_file <- tempfile(fileext = ".safetensors") + safetensors::safe_save_file(lapply(tensors, function(t) t$clone()), st_file) + handle <- safetensors::safetensors$new(st_file, framework = "torch") + model$weight_source <- list(keys = stats::setNames(names(tensors), names(tensors)), + handle = handle) + split_disk <- diffuseR:::.ltx23_block_stream(model, other_gb + 1.5 * blk_gb, + pin = FALSE, from_disk = TRUE) + expect_equal(split_disk$n_streamed, 1L) + w1 <- model$transformer_blocks[[1]]$parameters[[1]] + expect_equal(w1$stride(), rep(0L, w1$dim())) + diffuseR:::.staged_onload(split_disk$staging, "cpu") + res_disk <- run_model( + encoder_attention_mask = fx$m_enc_mask, + audio_encoder_attention_mask = fx$m_enc_mask + ) + expect_true(max_abs_diff(res_disk$sample, res$sample) == 0) + expect_true(max_abs_diff(res_disk$audio_sample, res$audio_sample) == 0) + diffuseR:::.staged_offload(split_disk$staging) + expect_equal(w1$stride(), rep(0L, w1$dim())) + # Back again for later tests + diffuseR:::.staged_onload(split_disk$staging, "cpu") + model$block_stream <- NULL + model$weight_source <- NULL + unlink(st_file) +} +# Disk staging needs the artifact keys only the NF4 loader records +expect_error(diffuseR:::.ltx23_block_stream(model, Inf, pin = FALSE, + from_disk = TRUE), "NF4") + # --- Key mapper ------------------------------------------------------------------ expect_equal( diff --git a/inst/tinytest/test_memory_ltx23.R b/inst/tinytest/test_memory_ltx23.R index cad3821..4629c38 100644 --- a/inst/tinytest/test_memory_ltx23.R +++ b/inst/tinytest/test_memory_ltx23.R @@ -67,6 +67,33 @@ expect_true(p8$stream_text) expect_true(ltx23_memory_profile(11)$stream_text) expect_equal(ltx23_memory_profile(4)$name, "cpu_only") +# Host RAM: plenty keeps everything in one process with host copies; +# less splits the text phase off, then drops the resident host copy +big <- ltx23_memory_profile(8, host_ram_gb = 64) +expect_false(big$split_text) +expect_equal(big$resident_from, "host") +mid <- ltx23_memory_profile(8, host_ram_gb = 24) +expect_true(mid$split_text) +expect_equal(mid$resident_from, "host") +small <- ltx23_memory_profile(12, host_ram_gb = 16) +expect_true(small$split_text) +expect_equal(small$resident_from, "disk") +# Unknown RAM keeps the defaults; the CPU tier never stages +unknown <- ltx23_memory_profile(8, host_ram_gb = NA) +expect_false(unknown$split_text) +expect_equal(unknown$resident_from, "host") +cpu <- ltx23_memory_profile(4, host_ram_gb = 8) +expect_false(cpu$split_text) +expect_equal(cpu$resident_from, "host") + # The loader refuses block streaming it cannot do expect_error(ltx23_load_pipeline("unused", device = "cpu", resident_gb = 4), "phase_offload") +expect_error(ltx23_load_pipeline("unused", device = "cpu", + resident_from = "disk"), "phase_offload") + +# The split text phase needs connectors, and an encoder or raw states +no_conn <- structure(list(), class = "ltx23_pipeline") +expect_error(ltx23_text_embeds("p", no_conn), "connectors") +fake <- structure(list(connectors = list()), class = "ltx23_pipeline") +expect_error(ltx23_text_embeds("p", fake), "prompt_embeds") diff --git a/man/ltx23_load_pipeline.Rd b/man/ltx23_load_pipeline.Rd index 5bde5f5..e839265 100644 --- a/man/ltx23_load_pipeline.Rd +++ b/man/ltx23_load_pipeline.Rd @@ -13,6 +13,7 @@ ltx23_load_pipeline( attn_chunk = NULL, phase_offload = TRUE, resident_gb = Inf, + resident_from = c("host", "disk"), verbose = TRUE ) } @@ -35,7 +36,10 @@ weights when loading the plain (non-fp8) checkpoint.} \item{components}{Character vector. Which components to load.} -\item{pin}{Logical. Pin fp8 host memory (fp8 artifact only).} +\item{pin}{Logical. Pin host memory: the fp8 weights, and the +phase-offload staging buffers (with \code{options(diffuseR.pin_staging)}). +Freed pinned memory stays in torch's host cache, so pass +\code{FALSE} for a pipeline that will be dropped to make room.} \item{attn_chunk}{Integer or NULL. Query-chunk size for attention (see \code{\link{ltx23_set_attn_chunk}}).} @@ -52,6 +56,14 @@ the ~12.5 GB NF4 transformer; under 7 GB the text connectors stream too. Needs \code{phase_offload} on CUDA. See \code{\link{ltx23_memory_profile}}.} +\item{resident_from}{Where the GPU-resident part of an NF4 +transformer comes back from after each render's decode phase: +\code{"host"} (default) keeps a pinned host copy, \code{"disk"} +keeps none and re-reads it from the artifact at the next onload +(the OS page cache serves it when RAM allows). \code{"disk"} saves +host RAM equal to the resident part (up to ~12 GB) for a re-read +per render. Needs \code{phase_offload} on CUDA.} + \item{verbose}{Logical.} } \value{ diff --git a/man/ltx23_memory_profile.Rd b/man/ltx23_memory_profile.Rd index bf7c26f..8aa1a51 100644 --- a/man/ltx23_memory_profile.Rd +++ b/man/ltx23_memory_profile.Rd @@ -3,15 +3,18 @@ \alias{ltx23_memory_profile} \title{Get an LTX-2.3 memory profile} \usage{ -ltx23_memory_profile(vram_gb = NULL) +ltx23_memory_profile(vram_gb = NULL, host_ram_gb = NULL) } \arguments{ \item{vram_gb}{Numeric or NULL (auto-detect free VRAM).} + +\item{host_ram_gb}{Numeric or NULL (auto-detect available RAM; NA +when undetectable, which keeps the all-in-RAM defaults).} } \value{ Named list with device/dtype placement, \code{attn_chunk}, - \code{pin_weights}, \code{resident_gb}, \code{stream_text}, and - resolution caps. + \code{pin_weights}, \code{resident_gb}, \code{stream_text}, + \code{split_text}, \code{resident_from}, and resolution caps. } \description{ Selects transformer precision, component placement, and attention @@ -40,4 +43,16 @@ streamed); fastest steps; about 9 percent weight round-trip error.} near-bf16 quality; each step pays the PCIe transfer.} } +Host RAM decides two more fields for the GPU tiers. \code{split_text} +asks the caller to encode in its own phase (Gemma, then the +connectors, each freed before the next loads; see +\code{\link{ltx23_text_embeds}}) instead of holding the encoder next +to the render pipeline. \code{resident_from} is passed to +\code{\link{ltx23_load_pipeline}}: \code{"disk"} drops the host copy +of the GPU-resident transformer blocks. Below 32 GB available the +text phase splits off; below 22 GB the resident blocks come from +disk (measured on LTX-2.5 at 768x512x121 with an 8 GB card: the +render phase peaks at 20.1 GB of host RAM with host copies and 16.0 +GB without, for ~8 s more per render). + } diff --git a/man/ltx23_text_embeds.Rd b/man/ltx23_text_embeds.Rd new file mode 100644 index 0000000..baceec7 --- /dev/null +++ b/man/ltx23_text_embeds.Rd @@ -0,0 +1,70 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_text_embeds} +\alias{ltx23_text_embeds} +\title{Encode an LTX prompt to connector embeddings} +\usage{ +ltx23_text_embeds( + prompt, + pipeline, + text_encoder = NULL, + tokenizer = NULL, + prompt_embeds = NULL, + device = "cuda", + dtype = "bfloat16", + max_sequence_length = 1024L, + verbose = TRUE +) +} +\arguments{ +\item{prompt}{Character. The prompt.} + +\item{pipeline}{An \code{ltx23_pipeline} holding the connectors, e.g. +\code{ltx23_load_pipeline(path, components = "connectors")}.} + +\item{prompt_embeds}{Optional raw encoder output (the result of +\code{\link{encode_with_gemma4}} or \code{\link{encode_with_gemma3}}), +so the encoder can be freed before the connectors load.} + +\item{device}{Character. Compute device.} + +\item{dtype}{Character. Compute dtype.} + +\item{max_sequence_length}{Integer. Text token length.} + +\item{verbose}{Logical.} + +\item{text_encoder,tokenizer}{The Gemma 3 or Gemma 4 model and +tokenizer (see \code{\link{txt2vid_ltx2}}). Ignored when +\code{prompt_embeds} is supplied.} +} +\value{ +A list with \code{video_text_embedding}, + \code{audio_text_embedding} and \code{attention_mask}, on the CPU. +} +\description{ +Runs the text encoder and the text connectors and returns their +output on the CPU, ready for the \code{connector_embeds} argument of +\code{\link{txt2vid_ltx2}}. This splits a render into phases that +never share host RAM: encode with Gemma and drop it, run a +connectors-only pipeline (loaded with \code{pin = FALSE}) and drop +it, then load the render pipeline without connectors. +} +\examples{ +\dontrun{ +ckpt <- file.path(tools::R_user_dir("diffuseR", "data"), "ltx2.5-nf4-full") +gemma <- file.path(tools::R_user_dir("diffuseR", "data"), "gemma4-nf4") +prompt <- "A red fox in fresh snow" +te <- load_gemma4_nf4(gemma, device = "cpu", stream = TRUE, pin = FALSE) +raw <- encode_with_gemma4(prompt, te, gemma4_tokenizer(gemma)) +rm(te); gc() +text <- ltx23_load_pipeline(ckpt, components = "connectors", + resident_gb = 4, pin = FALSE) +emb <- ltx23_text_embeds(prompt, text, prompt_embeds = raw) +rm(text, raw); gc() +pipe <- ltx23_load_pipeline(ckpt, components = c("dit", "vae", + "audio_vae", "vocoder"), resident_gb = 4, + resident_from = "disk") +txt2vid_ltx2(prompt, pipe, connector_embeds = emb, filename = "fox.mp4") +} + +}