From d47e59eefae0d610b4b701f7b329b407a617a1a6 Mon Sep 17 00:00:00 2001 From: Eliot McIntire Date: Sat, 14 Nov 2026 23:52:07 -0700 Subject: [PATCH 1/5] Build the ignition and spread covariates once per year, not at every same-year step Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014G18Mik3fH4z5M5CyMSUS6 --- NEWS.md | 2 + fireSense_dataPrepPredict.R | 56 +++++++++++++++++---- tests/testthat/test-covariatesOncePerYear.R | 46 +++++++++++++++++ 3 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 tests/testthat/test-covariatesOncePerYear.R diff --git a/NEWS.md b/NEWS.md index 7f32064..d5bd2e4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # fireSense_dataPrepPredict (development version) +* Inside one year (the reps of a hindcast, `fireTimeStep` < 1) the ignition and spread covariates are built once, not at every step. They are rebuilt when the year, the climate, `cohortData`, `pixelGroupMap`, `flammableRTM`, `landcoverDT`, `rstLCC_RTM` or the non-forest time since disturbance (or, for ignition, `lightningMaps`) changed, so a land-cover change inside a run is still seen (`covariatesBuilt()`). Same covariates; their `year` label is the first step's time. About 12 s per rep saved on a hindcast year. + * `dataYear` defaults to 2020 (was 2011), the default of Biomass_core, Biomass_borealDataPrep, Biomass_speciesData and burnSummaries, and may be up to 2025. `.inputObjects` stops when another module has a different `dataYear` (`SpaDES.core::paramCheckOtherMods()`). It is read only when `fireSense_dataPrepFit` does not supply the land cover, stand age and non-forest time since disturbance (a run that only predicts), so runs with `fireSense_dataPrepFit` predict as before. * A run without `fireSense_dataPrepFit` (one that only predicts) now works: `.inputObjects` reads the stored fits of the ELFs the study area touches from the SpreadFit ledger (`predictInputsFromLedger()`, with `fireSenseUtils::readSpreadFitLedger()`), sets the per-ELF `sppEquivs`, `nonForestedLCCGroupsList` and `missingLCCgroupList`, asks canClimateData for the fits' climate variables (`climateVariables`, `climateVariablesForFire`; the 'MDC' default made the ignition covariates fail with "[subset] invalid name(s)"), and makes `lightningMaps` (`fireSenseUtils::prepare_LightningData()`). `prepIgAndEscPredictData` makes `ignitionFitRTM`, the coarse grid of the ignition covariates, unless another module supplies it (`fireSense_ignitionPredict` stopped on a NULL one). New parameters `fitFileSuffix`, `spreadFitFilename`, `spreadFitGoogleDriveFolder` (as in `fireSense_dataPrepFit`). A `studyAreaWithSpreadParams` supplied as `NULL` predicts without stored fits, as before. diff --git a/fireSense_dataPrepPredict.R b/fireSense_dataPrepPredict.R index d672ac1..5404a4c 100644 --- a/fireSense_dataPrepPredict.R +++ b/fireSense_dataPrepPredict.R @@ -325,13 +325,14 @@ doEvent.fireSense_dataPrepPredict <- function(sim, eventTime, eventType) { ) }, prepIgAndEscPredictData = { - sim <- prepare_IgnitionAndEscapePredict(sim) + if (!covariatesBuilt(sim, "ignition", list(sim$lightningMaps[["lightningDays"]]))) + sim <- prepare_IgnitionAndEscapePredict(sim) sim <- scheduleEvent(sim, time(sim) + P(sim)$fireTimeStep, "fireSense_dataPrepPredict", "prepIgAndEscPredictData" ) }, prepSpreadPredictData = { - sim <- prepare_SpreadPredict(sim) + if (!covariatesBuilt(sim, "spread")) sim <- prepare_SpreadPredict(sim) sim <- scheduleEvent(sim, time(sim) + P(sim)$fireTimeStep, "fireSense_dataPrepPredict", "prepSpreadPredictData" ) @@ -734,6 +735,49 @@ prepare_SpreadPredict <- function(sim) { } +#' A stamp of the vegetation the fuel classes are made from +#' +#' Sums over `cohortData`, `pixelGroupMap` and `flammableRTM`: equal stamps mean the same inputs. +#' +#' @param sim A `simList`. +#' @return A list of numbers. +vegetationStamp <- function(sim) { + cd <- sim$cohortData + list(NROW(cd), sum(as.numeric(cd$B), na.rm = TRUE), sum(as.numeric(cd$age), na.rm = TRUE), + sum(as.numeric(cd$pixelGroup), na.rm = TRUE), + sum(terra::values(sim$pixelGroupMap, mat = FALSE), na.rm = TRUE), + sum(terra::values(sim$flammableRTM, mat = FALSE), na.rm = TRUE)) +} + +#' Are this year's covariates already built from these inputs? +#' +#' The ignition and spread covariates depend only on the year, the climate, the vegetation, the land cover +#' and the non-forest time since disturbance, so the steps inside one year (the reps of a hindcast, +#' `fireTimeStep` < 1) would rebuild the same tables. The stamp is of the actual inputs, not of time +#' alone: land cover and vegetation can change within a run. The stamp is kept in `mod` per `what`; it is +#' replaced when it differs, and `FALSE` is returned so the caller builds. +#' +#' @param sim A `simList`. +#' @param what `"ignition"` or `"spread"`: which covariates, and so which climate variables. +#' @param extra a list of further inputs of those covariates, stamped as they are. +#' +#' @return `TRUE` if the covariates in `sim` were built from these inputs, else `FALSE`. +covariatesBuilt <- function(sim, what, extra = list()) { + sumOf <- function(x) { + if (inherits(x, "SpatRaster")) return(if (terra::hasValues(x)) sum(terra::values(x, mat = FALSE), na.rm = TRUE)) + if (is.data.frame(x)) return(lapply(x, function(col) sum(as.numeric(col), na.rm = TRUE))) + x + } + climate <- sim$currentClimateRasters[[sim$climateVariablesForFire[[what]]]] + stamp <- list(floor(as.numeric(time(sim))), vegetationStamp(sim), sumOf(climate), + sumOf(sim$nonForest_timeSinceDisturbance), sumOf(sim$landcoverDT), sumOf(sim$rstLCC_RTM), + lapply(extra, sumOf)) + built <- sim[[c(ignition = "fireSense_igAndEscapePred_Covariates", spread = "fireSense_SpreadCovariates")[[what]]]] + if (!is.null(built) && identical(mod$covariateStamps[[what]], stamp)) return(TRUE) + mod$covariateStamps[[what]] <- stamp + FALSE +} + #' The fuel-class tables for this year, one per fuel set, shared by the ignition and spread covariates #' #' `fireSenseUtils::fireSenseCovariatesCreate()` is called for ignition (through @@ -752,13 +796,7 @@ prepare_SpreadPredict <- function(sim) { #' @return list of `data.table`s, one per element of `fuelSets`. fuelClassTablesThisYear <- function(sim, fuelSets) { year <- time(sim) - stamp <- function() { - cd <- sim$cohortData - list(NROW(cd), sum(as.numeric(cd$B), na.rm = TRUE), sum(as.numeric(cd$age), na.rm = TRUE), - sum(as.numeric(cd$pixelGroup), na.rm = TRUE), - sum(terra::values(sim$pixelGroupMap, mat = FALSE), na.rm = TRUE), - sum(terra::values(sim$flammableRTM, mat = FALSE), na.rm = TRUE)) - } + stamp <- function() vegetationStamp(sim) make <- function() lapply(fuelSets, function(fs) fireSenseUtils::cohortsToFuelClasses( cohortData = sim$cohortData, pixelGroupMap = sim$pixelGroupMap, diff --git a/tests/testthat/test-covariatesOncePerYear.R b/tests/testthat/test-covariatesOncePerYear.R new file mode 100644 index 0000000..840054e --- /dev/null +++ b/tests/testthat/test-covariatesOncePerYear.R @@ -0,0 +1,46 @@ +## Inside one year (the reps of a hindcast, `fireTimeStep` < 1) the covariates do not change, so they are +## built once. A change to an input they are made from (land cover, vegetation, climate) rebuilds them. + +countFuelClassCalls <- function(code) { + n <- 0L + orig <- fireSenseUtils::cohortsToFuelClasses + testthat::local_mocked_bindings( + cohortsToFuelClasses = function(...) { n <<- n + 1L; orig(...) }, + .package = "fireSenseUtils") + force(code) + n +} + +repTimes <- list(start = 2001, end = 2001.75) +repParams <- function(what) list(fireTimeStep = 0.25, whichModulesToPrepare = what) + +for (what in c("fireSense_ignitionPredict", "fireSense_spreadPredict")) { + covName <- if (what == "fireSense_ignitionPredict") "fireSense_igAndEscapePred_Covariates" else "fireSense_SpreadCovariates" + + test_that(paste("four same-year steps build the covariates once, as one step does:", what), { + n <- countFuelClassCalls(reps <- toyPrepRun(toyObjects(), times = repTimes, params = repParams(what))) + once <- toyPrepRun(toyObjects(), times = list(start = 2001, end = 2001), + params = list(whichModulesToPrepare = what)) + ## the tables are the first step's (their `year` label is 2001), not rebuilt at 2001.75 + expect_identical(n, 1L) # was 4 + expect_identical(covDF(reps[[covName]]), covDF(once[[covName]])) + }) + + test_that(paste("a vegetation change between steps of one year rebuilds the covariates:", what), { + sim <- SpaDES.core::spades(toyPrepSim(toyObjects(), times = list(start = 2001, end = 2001), + params = repParams(what)), debug = FALSE) + first <- data.table::copy(sim[[covName]]) + sim$cohortData$B <- sim$cohortData$B * 2L + ## the three more steps of the year: the first rebuilds, the others reuse it + SpaDES.core::end(sim) <- 2001.75 + n <- countFuelClassCalls(sim <- SpaDES.core::spades(sim, debug = FALSE)) + expect_identical(n, 1L) + expect_false(isTRUE(all.equal(covDF(first), covDF(sim[[covName]])))) + }) +} + +test_that("a new year rebuilds the covariates", { + n <- countFuelClassCalls(toyPrepRun(toyObjects(), times = list(start = 2001, end = 2003), + params = list(whichModulesToPrepare = "fireSense_spreadPredict"))) + expect_identical(n, 3L) +}) From 7fbc39264f4fa83207afe941c134ec213d37fe13 Mon Sep 17 00:00:00 2001 From: Eliot McIntire Date: Sat, 14 Nov 2026 23:59:43 -0700 Subject: [PATCH 2/5] replaySave and replayFrom: save the spread-prediction inputs of a hindcast job and load them in a replay job Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014G18Mik3fH4z5M5CyMSUS6 --- NEWS.md | 2 + fireSense_dataPrepPredict.R | 108 ++++++++++++++++++++++++++++++++- tests/testthat/test-init.R | 7 ++- tests/testthat/test-metadata.R | 13 +++- tests/testthat/test-replay.R | 77 +++++++++++++++++++++++ 5 files changed, 203 insertions(+), 4 deletions(-) create mode 100644 tests/testthat/test-replay.R diff --git a/NEWS.md b/NEWS.md index d5bd2e4..c60f1df 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # fireSense_dataPrepPredict (development version) +* New parameters `replaySave` (default `FALSE`) and `replayFrom` (default `NA`): no change unless set. `replaySave = TRUE` writes, once after the first covariate build, the spread-prediction inputs that do not depend on the spread parameters to `outputPath(sim)`: `fireSense_SpreadCovariates`, `flammableRTM`, `rasterToMatch`, `rasterToMatchELF`, `rasterToMatchLargeELF`, `studyAreaWithSpreadParams`, `nonEscapedFireSizesHa`, `ignitionFirePoints` and `firePolys` (those present): `replay_.tif` and `replay_.gpkg` for terra objects (`terra::writeRaster()`, `terra::writeVector()`), the rest in `replay_spreadInputs.qs2`. `replayFrom = ` loads them in `.inputObjects` and schedules no covariate event, so a job needs no climate, vegetation, fit or ELF module. `qs2` is now in `reqdPkgs`. + * Inside one year (the reps of a hindcast, `fireTimeStep` < 1) the ignition and spread covariates are built once, not at every step. They are rebuilt when the year, the climate, `cohortData`, `pixelGroupMap`, `flammableRTM`, `landcoverDT`, `rstLCC_RTM` or the non-forest time since disturbance (or, for ignition, `lightningMaps`) changed, so a land-cover change inside a run is still seen (`covariatesBuilt()`). Same covariates; their `year` label is the first step's time. About 12 s per rep saved on a hindcast year. * `dataYear` defaults to 2020 (was 2011), the default of Biomass_core, Biomass_borealDataPrep, Biomass_speciesData and burnSummaries, and may be up to 2025. `.inputObjects` stops when another module has a different `dataYear` (`SpaDES.core::paramCheckOtherMods()`). It is read only when `fireSense_dataPrepFit` does not supply the land cover, stand age and non-forest time since disturbance (a run that only predicts), so runs with `fireSense_dataPrepFit` predict as before. diff --git a/fireSense_dataPrepPredict.R b/fireSense_dataPrepPredict.R index 5404a4c..d75cd15 100644 --- a/fireSense_dataPrepPredict.R +++ b/fireSense_dataPrepPredict.R @@ -26,6 +26,7 @@ defineModule(sim, list( "PredictiveEcology/LandR@development", "PredictiveEcology/reproducible@development", "PredictiveEcology/SpaDES.core@development (>= 3.0.4)", + "qs2", "sf", "terra" ), @@ -78,6 +79,16 @@ defineModule(sim, list( ), defineParameter("nonForestCanBeYoungAge", "logical", fireSenseUtils::fireSenseNonForestCanBeYoungAge, NA, NA, desc = "Should burned non-forest pixels be `youngAge` until `cutoffForYoungAge`?"), + defineParameter("replayFrom", "character", NA_character_, NA, NA, + desc = paste("Path of a reference hindcast job's `outputPath` (one ELF x year folder) made with", + "`replaySave = TRUE`. When set, `.inputObjects` loads the spread-prediction inputs", + "that do not depend on the spread parameters from it (`replayObjects`) and no", + "covariate events are scheduled. `NA`: build as usual. The same name is a parameter", + "of `fireSense_burn`, so `.globals = list(replayFrom = ...)` sets both.")), + defineParameter("replaySave", "logical", FALSE, NA, NA, + desc = paste("Write those inputs (`replay_spreadInputs.qs2`, `replay_.tif`,", + "`replay_.gpkg`) to `outputPath(sim)` once, after the first covariate build,", + "for a later job's `replayFrom`. For a job of one year.")), defineParameter("scanfiVersion", "character", fireSenseUtils::fireSenseSCANFIVersion, NA, NA, desc = paste( "SCANFI land-cover version for non-forest land cover, when this module builds its own", @@ -176,6 +187,14 @@ defineModule(sim, list( "'year'. Only used if `currentClimateRasters` is not supplied.")), expectsInput("rasterToMatch", "SpatRaster", sourceURL = NA, desc = "Template raster for the study area."), + expectsInput("rasterToMatchELF", "SpatRaster", sourceURL = NA, + desc = "From `fireSense_ELFs`. Only saved and replayed (`replaySave`, `replayFrom`)."), + expectsInput("nonEscapedFireSizesHa", "numeric", sourceURL = NA, + desc = "From `fireSense_dataPrepFit`. Only saved and replayed (`replaySave`, `replayFrom`)."), + expectsInput("ignitionFirePoints", "SpatVector", sourceURL = NA, + desc = "From `fireSense_dataPrepFit`. Only saved and replayed (`replaySave`, `replayFrom`)."), + expectsInput("firePolys", "list", sourceURL = NA, + desc = "Annual fire polygons, if supplied. Only saved and replayed (`replaySave`, `replayFrom`)."), expectsInput("rasterToMatchLargeELF", "SpatRaster", sourceURL = NA, desc = paste("From `fireSense_ELFs`. A neighbour ELF is blended (`blendNeighbourELFs`) only if this", "raster labels its pixels, which is how `fireSense_spreadPredict` weights each ELF.")), @@ -256,6 +275,13 @@ defineModule(sim, list( desc = paste( "Ignition and escape covariates at the aggregated (`igAggFactor`) resolution;", "`pixelID` is the cell index of the aggregated raster.")), + ## with `replayFrom`, set by `.inputObjects` (declared so the modules that read them are set up after this one) + createsOutput("rasterToMatch", "SpatRaster", desc = "With `replayFrom`: loaded by `.inputObjects`."), + createsOutput("rasterToMatchELF", "SpatRaster", desc = "With `replayFrom`: loaded by `.inputObjects`."), + createsOutput("rasterToMatchLargeELF", "SpatRaster", desc = "With `replayFrom`: loaded by `.inputObjects`."), + createsOutput("nonEscapedFireSizesHa", "numeric", desc = "With `replayFrom`: loaded by `.inputObjects`."), + createsOutput("ignitionFirePoints", "SpatVector", desc = "With `replayFrom`: loaded by `.inputObjects`."), + createsOutput("firePolys", "list", desc = "With `replayFrom`: loaded by `.inputObjects`."), createsOutput("fireSense_SpreadCovariates", "data.table", desc = "Spread covariates; `pixelID` is the cell index of `flammableRTM`."), createsOutput("nonForest_timeSinceDisturbance", "SpatRaster", @@ -288,6 +314,7 @@ doEvent.fireSense_dataPrepPredict <- function(sim, eventTime, eventType) { if ("fireSense_EscapePredict" %in% P(sim)$whichModulesToPrepare) stop("fireSense_EscapePredict no longer exists as a module; escape covariates are prepared with ", "fireSense_ignitionPredict. Remove it from parameter whichModulesToPrepare.") + if (isReplay(sim)) return(invisible(sim)) # `.inputObjects` loaded everything; nothing to build sim <- Init(sim) sim <- scheduleEvent(sim, time(sim) + P(sim)$fireTimeStep, "fireSense_dataPrepPredict", "ageNonForest") sim <- scheduleEvent(sim, P(sim)$.runInitialTime, "fireSense_dataPrepPredict", "getClimateRasters") @@ -302,6 +329,9 @@ doEvent.fireSense_dataPrepPredict <- function(sim, eventTime, eventType) { sim <- scheduleEvent(sim, P(sim)$.runInitialTime, "fireSense_dataPrepPredict", "prepSpreadPredictData" ) } + ## after the first build (priority 5) + if (isTRUE(P(sim)$replaySave)) + sim <- scheduleEvent(sim, P(sim)$.runInitialTime, "fireSense_dataPrepPredict", "save", eventPriority = 7) }, ageNonForest = { ## time since disturbance is in whole years: it advances by the calendar years passed since the last @@ -338,7 +368,11 @@ doEvent.fireSense_dataPrepPredict <- function(sim, eventTime, eventType) { ) }, save = { - message("fireSense_dataPrepPredict: the save event does nothing") + if (isTRUE(P(sim)$replaySave)) { + saveReplay(mget(replayObjects, envir = envir(sim), ifnotfound = list(NULL)), outputPath(sim)) + } else { + message("fireSense_dataPrepPredict: the save event does nothing") + } }, warning(paste("Undefined event type: \'", current(sim)[1, "eventType", with = FALSE], "\' in module \'", current(sim)[1, "moduleName", with = FALSE], "\'", @@ -1110,6 +1144,11 @@ predictInputsFromLedger <- function(sim) { #' #' @return The `simList`, invisibly. .inputObjects <- function(sim) { + if (isReplay(sim)) { # a replay job: the saved inputs, nothing built + message(currentModule(sim), ": replaying the inputs saved in '", P(sim)$replayFrom, "'.") + list2env(loadReplay(P(sim)$replayFrom), envir(sim)) + return(invisible(sim)) + } cacheTags <- c(currentModule(sim), "otherFunctions:.inputObjects") dPath <- asPath(inputPath(sim), 1) ## the same data year as the vegetation modules (Biomass_core, Biomass_borealDataPrep, ...) @@ -1192,6 +1231,73 @@ predictInputsFromLedger <- function(sim) { return(invisible(sim)) } +## The spread-prediction inputs that do not depend on the spread parameters: what a replay job needs in place +## of the climate, vegetation, fit and ELF modules. This module builds `fireSense_SpreadCovariates` (and +## `flammableRTM` from `fireSense_dataPrepFit`'s per-year list, `studyAreaWithSpreadParams` from the ledger +## when `fireSense_dataPrepFit` is absent); `fireSense_ELFs` makes the three `rasterToMatch*`, +## `fireSense_dataPrepFit` makes `nonEscapedFireSizesHa` and `ignitionFirePoints`, and `firePolys` is supplied +## by the user (or absent). +replayObjects <- c("fireSense_SpreadCovariates", "flammableRTM", "rasterToMatch", "rasterToMatchELF", + "rasterToMatchLargeELF", "studyAreaWithSpreadParams", "nonEscapedFireSizesHa", + "ignitionFirePoints", "firePolys") + +isReplay <- function(sim) !is.na(P(sim)$replayFrom) + +replayFile <- function(path, name, ext) file.path(path, paste0("replay_", name, ".", ext)) + +## how an object is written: terra objects with writeRaster()/writeVector() (never saveRDS), the rest qs2 +replayKind <- function(x) { + if (inherits(x, "SpatRaster")) "raster" + else if (inherits(x, "SpatVector")) "vector" + else if (inherits(x, "sf")) "sf" + else if (is.list(x) && length(x) && all(vapply(x, inherits, logical(1), "SpatVector"))) "vectors" + else "qs2" +} + +#' Write the replay inputs +#' +#' @param objs named list of the objects (`NULL`s are not written). +#' @param path folder to write in: `replay_.tif` (`SpatRaster`), `replay_.gpkg` (`SpatVector`, `sf`; +#' `replay__.gpkg` for a list of them) and `replay_spreadInputs.qs2` (the other objects, and how each +#' object was written). +#' @return `path`, invisibly. +saveReplay <- function(objs, path) { + objs <- Filter(Negate(is.null), objs) + kinds <- vapply(objs, replayKind, character(1)) + dir.create(path, recursive = TRUE, showWarnings = FALSE) + for (nm in names(objs)) { + x <- objs[[nm]] + switch(kinds[[nm]], + raster = terra::writeRaster(x, replayFile(path, nm, "tif"), overwrite = TRUE), + vector = terra::writeVector(x, replayFile(path, nm, "gpkg"), overwrite = TRUE), + sf = terra::writeVector(terra::vect(x), replayFile(path, nm, "gpkg"), overwrite = TRUE), + vectors = for (i in seq_along(x)) + terra::writeVector(x[[i]], replayFile(path, paste0(nm, "_", i), "gpkg"), overwrite = TRUE)) + } + qs2::qs_save(c(list(kinds = kinds), objs[kinds == "qs2"]), replayFile(path, "spreadInputs", "qs2")) + invisible(path) +} + +#' Read the replay inputs written by `saveReplay()` +#' +#' @param path folder `saveReplay()` wrote to. +#' @return named list of the objects. +loadReplay <- function(path) { + bundle <- qs2::qs_read(replayFile(path, "spreadInputs", "qs2")) + kinds <- bundle$kinds + out <- bundle[setdiff(names(bundle), "kinds")] + for (nm in names(kinds)[kinds != "qs2"]) + out[[nm]] <- switch(kinds[[nm]], + raster = terra::rast(replayFile(path, nm, "tif")), + vector = terra::vect(replayFile(path, nm, "gpkg")), + sf = sf::st_as_sf(terra::vect(replayFile(path, nm, "gpkg"))), + vectors = { + files <- sort(list.files(path, pattern = paste0("^replay_", nm, "_[0-9]+\\.gpkg$"), full.names = TRUE)) + lapply(files[order(as.integer(sub(".*_([0-9]+)\\.gpkg$", "\\1", files)))], terra::vect) + }) + out[names(kinds)] +} + ## An NA `.studyAreaName` becomes a hash of `studyArea`, as in Biomass_borealDataPrep. Without a ## `studyArea` it stays NA (see PredictiveEcology/LandR#246). resolveStudyAreaName <- function(sim) { diff --git a/tests/testthat/test-init.R b/tests/testthat/test-init.R index cc76cf2..dc341d9 100644 --- a/tests/testthat/test-init.R +++ b/tests/testthat/test-init.R @@ -5,8 +5,11 @@ test_that("init produces every declared output on the toy landscape", { md <- SpaDES.core::moduleMetadata(module = moduleName, path = modulePath) ## studyAreaWithSpreadParams is an output only to be restored on a cache hit; this run supplies none. ## The others are set only when .inputObjects reads the SpreadFit ledger, which this run does not. + ## The replay objects are outputs only so the modules that read them are set up after this one: they are set + ## only with `replayFrom`. fromLedger <- c("studyAreaWithSpreadParams", "climateVariables", "sppEquivs", "nonForestedLCCGroupsList", - "missingLCCgroupList") + "missingLCCgroupList", "rasterToMatchELF", "rasterToMatchLargeELF", "nonEscapedFireSizesHa", + "ignitionFirePoints", "firePolys") for (obj in setdiff(md$outputObjects$objectName, fromLedger)) { expect_false(is.null(sim[[obj]]), info = obj) } @@ -106,7 +109,7 @@ test_that("the retired fireSense_EscapePredict stops with a message naming fireS "no longer exists.*fireSense_ignitionPredict") }) -test_that("the save event does nothing", { +test_that("the save event does nothing without replaySave", { sim <- SpaDES.core::spades(toyPrepSim(), events = "init", debug = FALSE) before <- sapply(ls(sim), function(nm) reproducible::.robustDigest(sim[[nm]])) queued <- as.data.frame(SpaDES.core::events(sim)) diff --git a/tests/testthat/test-metadata.R b/tests/testthat/test-metadata.R index 8de4bba..feeb065 100644 --- a/tests/testthat/test-metadata.R +++ b/tests/testthat/test-metadata.R @@ -24,13 +24,16 @@ test_that("inputs are the expected names and classes", { cohortData = "data.table", cohortDatas = "list", currentClimateRasters = "SpatRaster", + firePolys = "list", flammableRTM = "SpatRaster", flammableRTMs = "list", + ignitionFirePoints = "SpatVector", landcoverDT = "data.table", landcoverDTs = "list", lightningMaps = "SpatRaster", missingLCCgroup = "character", missingLCCgroupList = "list", + nonEscapedFireSizesHa = "numeric", nonForest_timeSinceDisturbances = "list", nonForestedLCCGroups = "list", nonForestedLCCGroupsList = "list", @@ -38,6 +41,7 @@ test_that("inputs are the expected names and classes", { pixelGroupMaps = "list", projectedClimateRasters = "list", rasterToMatch = "SpatRaster", + rasterToMatchELF = "SpatRaster", rasterToMatchLargeELF = "SpatRaster", rstCurrentBurn = "SpatRaster", rstLCC_RTM = "SpatRaster", @@ -60,16 +64,22 @@ test_that("outputs are the expected names and classes", { climateVariablesForFire = "list", cohortData = "data.table", currentClimateRasters = "SpatRaster", + firePolys = "list", fireSense_igAndEscapePred_Covariates = "data.table", fireSense_SpreadCovariates = "data.table", flammableRTM = "SpatRaster", + ignitionFirePoints = "SpatVector", ignitionFitRTM = "SpatRaster", landcoverDT = "data.table", lightningMaps = "SpatRaster", missingLCCgroupList = "list", + nonEscapedFireSizesHa = "numeric", nonForest_timeSinceDisturbance = "SpatRaster", nonForestedLCCGroupsList = "list", pixelGroupMap = "SpatRaster", + rasterToMatch = "SpatRaster", + rasterToMatchELF = "SpatRaster", + rasterToMatchLargeELF = "SpatRaster", rstLCC_RTM = "SpatRaster", sppEquivs = "list", standAgeMap = "SpatRaster", @@ -84,7 +94,8 @@ test_that("parameters are the expected names", { sort(c(".runInitialTime", ".studyAreaName", ".useCache", "blendNeighbourELFs", "cutoffForYoungAge", "dataYear", "fireTimeStep", "fitFileSuffix", "flammabilityThreshold", "forestedLCC", "fuelClassCol", "igAggFactor", "nonflammableLCC", "nonForestCanBeYoungAge", "scanfiVersion", - "spreadFitFilename", "spreadFitGoogleDriveFolder", "sppEquivCol", "whichModulesToPrepare")) + "spreadFitFilename", "spreadFitGoogleDriveFolder", "sppEquivCol", "whichModulesToPrepare", + "replayFrom", "replaySave")) ) }) diff --git a/tests/testthat/test-replay.R b/tests/testthat/test-replay.R new file mode 100644 index 0000000..2339bb9 --- /dev/null +++ b/tests/testthat/test-replay.R @@ -0,0 +1,77 @@ +## A replay job (`replayFrom`) takes the spread-prediction inputs that do not depend on the spread parameters +## from a reference hindcast job (`replaySave = TRUE`), in place of building them. + +replayObjs <- function() { + pts <- terra::vect(cbind(x = c(100, 600), y = c(200, 900)), type = "points", crs = toyCRSp) + pts$escaped <- c(TRUE, FALSE) + poly <- terra::as.polygons(terra::ext(0, 500, 0, 500), crs = toyCRSp) + poly$YEAR <- 2000L + sfRow <- sf::st_as_sf(poly) + sfRow$ELF <- "X" + sfRow$p1 <- 0.5 + list( + flammableRTM = toyRastP(c(rep(1, 15), 0), "flammable"), + rasterToMatch = toyRastP(rep(1L, 16), "rtm"), + rasterToMatchELF = toyRastP(rep(c(1L, 2L), 8), "ELF"), + rasterToMatchLargeELF = toyRastP(rep(2L, 16), "large"), + fireSense_SpreadCovariates = data.table::data.table(pixelID = 1:3, MDC = c(1, 2, 3), youngAge = c(0, 1, 0)), + studyAreaWithSpreadParams = sfRow, + nonEscapedFireSizesHa = c(1.5, 20, 300), + ignitionFirePoints = pts, + firePolys = list(poly, poly) + ) +} + +## SpatRaster/SpatVector are pointers: compare what they hold +asData <- function(x) { + if (inherits(x, "SpatRaster")) list(names(x), terra::values(x, mat = FALSE), as.vector(terra::ext(x)), terra::crs(x)) + else if (inherits(x, "SpatVector")) list(as.data.frame(x), terra::geom(x), terra::crs(x)) + else if (is.data.frame(x)) as.data.frame(x) + else if (is.list(x)) lapply(x, asData) + else x +} + +test_that("without replaySave nothing is written, and the parameters default to no replay", { + ref <- toyPrepRun(toyObjects(), params = list(whichModulesToPrepare = "fireSense_spreadPredict")) + expect_length(list.files(SpaDES.core::outputPath(ref), pattern = "^replay_"), 0L) + md <- SpaDES.core::moduleMetadata(module = moduleName, path = modulePath) + expect_true(is.na(md$parameters[md$parameters$paramName == "replayFrom", ]$default[[1]])) + expect_false(md$parameters[md$parameters$paramName == "replaySave", ]$default[[1]]) +}) + +test_that("saved replay inputs load back identical", { + objs <- replayObjs() + path <- withr::local_tempdir() + saveReplay(objs, path) + expect_true(file.exists(file.path(path, "replay_spreadInputs.qs2"))) + expect_true(file.exists(file.path(path, "replay_flammableRTM.tif"))) + back <- loadReplay(path) + expect_setequal(names(back), names(objs)) + for (nm in setdiff(names(objs), "studyAreaWithSpreadParams")) + expect_identical(asData(back[[nm]]), asData(objs[[nm]]), info = nm) + expect_s3_class(back$studyAreaWithSpreadParams, "sf") + expect_equal(sf::st_drop_geometry(back$studyAreaWithSpreadParams), sf::st_drop_geometry(objs$studyAreaWithSpreadParams), + ignore_attr = TRUE) +}) + +test_that("a replay job takes the saved inputs and builds none", { + objs <- toyInputsP() + ref <- toyPrepRun(utils::modifyList(toyObjects(), replayObjs()[c("rasterToMatchELF", "rasterToMatchLargeELF", + "nonEscapedFireSizesHa", "ignitionFirePoints", "firePolys")]), + params = list(whichModulesToPrepare = "fireSense_spreadPredict", replaySave = TRUE)) + path <- SpaDES.core::outputPath(ref) + expect_true(file.exists(file.path(path, "replay_spreadInputs.qs2"))) + n <- 0L + orig <- fireSenseUtils::cohortsToFuelClasses + testthat::local_mocked_bindings(cohortsToFuelClasses = function(...) { n <<- n + 1L; orig(...) }, + .package = "fireSenseUtils") + rep <- toyPrepRun(list(), params = list(whichModulesToPrepare = "fireSense_spreadPredict", replayFrom = path)) + expect_identical(n, 0L) + expect_identical(covDF(rep$fireSense_SpreadCovariates), covDF(ref$fireSense_SpreadCovariates)) + for (nm in c("flammableRTM", "rasterToMatch", "rasterToMatchELF", "rasterToMatchLargeELF", "nonEscapedFireSizesHa", + "ignitionFirePoints", "firePolys")) + expect_identical(asData(rep[[nm]]), asData(ref[[nm]]), info = nm) + ## and no build events were scheduled + expect_false(any(c("getClimateRasters", "prepSpreadPredictData") %in% SpaDES.core::completed(rep)$eventType)) +}) + From c59d1782a19dc8722a9926ead8007e3aa615590d Mon Sep 17 00:00:00 2001 From: Eliot McIntire Date: Sun, 15 Nov 2026 00:17:52 -0700 Subject: [PATCH 3/5] Stamp the fitted rows and use digests; refresh the per-ELF land cover when it changes Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014G18Mik3fH4z5M5CyMSUS6 --- NEWS.md | 3 +- fireSense_dataPrepPredict.R | 60 +++++++++++++-------- tests/testthat/test-covariatesOncePerYear.R | 57 ++++++++++++++++++++ 3 files changed, 98 insertions(+), 22 deletions(-) diff --git a/NEWS.md b/NEWS.md index d5bd2e4..5ffa890 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,7 @@ # fireSense_dataPrepPredict (development version) -* Inside one year (the reps of a hindcast, `fireTimeStep` < 1) the ignition and spread covariates are built once, not at every step. They are rebuilt when the year, the climate, `cohortData`, `pixelGroupMap`, `flammableRTM`, `landcoverDT`, `rstLCC_RTM` or the non-forest time since disturbance (or, for ignition, `lightningMaps`) changed, so a land-cover change inside a run is still seen (`covariatesBuilt()`). Same covariates; their `year` label is the first step's time. About 12 s per rep saved on a hindcast year. +* Inside one year (the reps of a hindcast, `fireTimeStep` < 1) the ignition and spread covariates are built once, not at every step. They are rebuilt when the year, the climate, `cohortData`, `pixelGroupMap`, `flammableRTM`, `landcoverDT`, `rstLCC_RTM`, the non-forest time since disturbance, `lightningMaps` (ignition) or the fitted rows (`studyAreaWithSpreadParams`: ELF, polygons, fitted parameter names) changed; inputs are compared by xxhash64 of their values (`covariatesBuilt()`, `inputDigest()`). Same covariates; their `year` label is the first step's time. About 12 s per rep saved on a hindcast year. +* The land cover of the per-ELF fuel sets (`mod$ELFrstLCC`, `mod$ELFlandcoverDTs`) was made once per run and never refreshed, so a change of `rstLCC_RTM` or `flammableRTM` inside a run was not seen; it is now made again when either changed. `digest` is in `reqdPkgs`. * `dataYear` defaults to 2020 (was 2011), the default of Biomass_core, Biomass_borealDataPrep, Biomass_speciesData and burnSummaries, and may be up to 2025. `.inputObjects` stops when another module has a different `dataYear` (`SpaDES.core::paramCheckOtherMods()`). It is read only when `fireSense_dataPrepFit` does not supply the land cover, stand age and non-forest time since disturbance (a run that only predicts), so runs with `fireSense_dataPrepFit` predict as before. diff --git a/fireSense_dataPrepPredict.R b/fireSense_dataPrepPredict.R index 5404a4c..e4fd8da 100644 --- a/fireSense_dataPrepPredict.R +++ b/fireSense_dataPrepPredict.R @@ -20,6 +20,7 @@ defineModule(sim, list( reqdPkgs = list( "PredictiveEcology/climateData@development", # latestHistoricalYear() "data.table", + "digest", "googledrive", "PredictiveEcology/fireSenseUtils@development (>= 0.2.3.9142)", # 9142: readSpreadFitLedger() and the other predict-only helpers (#206); fuelCovsToCoarse() hands back the static it used (#191) "httr2", @@ -735,43 +736,53 @@ prepare_SpreadPredict <- function(sim) { } +#' A digest of an input, to tell whether it changed +#' +#' xxhash64 of the values (a `SpatRaster`: its names and values; a table: its columns), so a change that keeps +#' the sum (two pixels swapped) is seen too. +#' +#' @param x a `SpatRaster`, table, list or `NULL`. +#' @return a character. +inputDigest <- function(x) { + if (inherits(x, "SpatRaster")) x <- if (terra::hasValues(x)) list(names(x), terra::values(x, mat = FALSE)) + if (is.data.frame(x)) x <- as.list(x) + digest::digest(x, algo = "xxhash64") +} + #' A stamp of the vegetation the fuel classes are made from #' -#' Sums over `cohortData`, `pixelGroupMap` and `flammableRTM`: equal stamps mean the same inputs. +#' Digests of `cohortData`, `pixelGroupMap` and `flammableRTM`: equal stamps mean the same inputs. #' #' @param sim A `simList`. -#' @return A list of numbers. +#' @return A list of digests. vegetationStamp <- function(sim) { - cd <- sim$cohortData - list(NROW(cd), sum(as.numeric(cd$B), na.rm = TRUE), sum(as.numeric(cd$age), na.rm = TRUE), - sum(as.numeric(cd$pixelGroup), na.rm = TRUE), - sum(terra::values(sim$pixelGroupMap, mat = FALSE), na.rm = TRUE), - sum(terra::values(sim$flammableRTM, mat = FALSE), na.rm = TRUE)) + lapply(list(sim$cohortData, sim$pixelGroupMap, sim$flammableRTM), inputDigest) } #' Are this year's covariates already built from these inputs? #' -#' The ignition and spread covariates depend only on the year, the climate, the vegetation, the land cover -#' and the non-forest time since disturbance, so the steps inside one year (the reps of a hindcast, -#' `fireTimeStep` < 1) would rebuild the same tables. The stamp is of the actual inputs, not of time -#' alone: land cover and vegetation can change within a run. The stamp is kept in `mod` per `what`; it is -#' replaced when it differs, and `FALSE` is returned so the caller builds. +#' The ignition and spread covariates depend only on the year, the climate, the vegetation, the land cover, +#' the non-forest time since disturbance and the fitted rows (which ELF, which polygons, which fitted +#' parameter names, from which the fuel-class roles are read), so the steps inside one year (the reps of a +#' hindcast, `fireTimeStep` < 1) would rebuild the same tables. The stamp is of the actual inputs, not of +#' time alone: land cover and vegetation can change within a run. It is kept in `mod` per `what`; it is +#' replaced when it differs, and `FALSE` is returned so the caller builds. The rows are reduced to the own +#' ELF's first (`useOwnELFRows()`, as the builders do), so the stamp sees what the builders see. #' #' @param sim A `simList`. #' @param what `"ignition"` or `"spread"`: which covariates, and so which climate variables. -#' @param extra a list of further inputs of those covariates, stamped as they are. +#' @param extra a list of further inputs of those covariates. #' #' @return `TRUE` if the covariates in `sim` were built from these inputs, else `FALSE`. covariatesBuilt <- function(sim, what, extra = list()) { - sumOf <- function(x) { - if (inherits(x, "SpatRaster")) return(if (terra::hasValues(x)) sum(terra::values(x, mat = FALSE), na.rm = TRUE)) - if (is.data.frame(x)) return(lapply(x, function(col) sum(as.numeric(col), na.rm = TRUE))) - x - } + sim <- useOwnELFRows(sim) + sa <- sim$studyAreaWithSpreadParams + rows <- list(as.character(sim$.ELFind), as.character(sa[[fireSenseUtils::polygonIDTxt]]), + if (!is.null(sa$params)) lapply(sa$params, colnames)) climate <- sim$currentClimateRasters[[sim$climateVariablesForFire[[what]]]] - stamp <- list(floor(as.numeric(time(sim))), vegetationStamp(sim), sumOf(climate), - sumOf(sim$nonForest_timeSinceDisturbance), sumOf(sim$landcoverDT), sumOf(sim$rstLCC_RTM), - lapply(extra, sumOf)) + stamp <- list(floor(as.numeric(time(sim))), vegetationStamp(sim), + lapply(c(list(climate, sim$nonForest_timeSinceDisturbance, sim$landcoverDT, sim$rstLCC_RTM), extra), + inputDigest), rows) built <- sim[[c(ignition = "fireSense_igAndEscapePred_Covariates", spread = "fireSense_SpreadCovariates")[[what]]]] if (!is.null(built) && identical(mod$covariateStamps[[what]], stamp)) return(TRUE) mod$covariateStamps[[what]] <- stamp @@ -990,6 +1001,13 @@ useOwnELFRows <- function(sim) { ELFfuelSets <- function(sim) { fcc <- P(sim)$fuelClassCol + ## the land cover the fuel sets are made from is kept, and made again when `rstLCC_RTM` or `flammableRTM` changed + lcStamp <- lapply(list(sim$rstLCC_RTM, sim$flammableRTM), inputDigest) + if (!identical(mod$ELFlcStamp, lcStamp)) { + mod$ELFrstLCC <- NULL + mod$ELFlandcoverDTs <- NULL + mod$ELFlcStamp <- lcStamp + } if (length(sim$sppEquivs) > 1L || length(sim$nonForestedLCCGroupsList)) { n <- length(sim$sppEquivs) if (length(sim$nonForestedLCCGroupsList) != n || length(sim$missingLCCgroupList) != n) diff --git a/tests/testthat/test-covariatesOncePerYear.R b/tests/testthat/test-covariatesOncePerYear.R index 840054e..476b4b6 100644 --- a/tests/testthat/test-covariatesOncePerYear.R +++ b/tests/testthat/test-covariatesOncePerYear.R @@ -44,3 +44,60 @@ test_that("a new year rebuilds the covariates", { params = list(whichModulesToPrepare = "fireSense_spreadPredict"))) expect_identical(n, 3L) }) + +## the first step of the year, then `change(sim)`, then the other three: the number of fuel-class builds +## they make, and the covariates before and after +stepsAfterChange <- function(objs, change, what, params = NULL) { + p <- c(list(fireTimeStep = 0.25), if (!is.null(what)) list(whichModulesToPrepare = what), params) + sim <- SpaDES.core::spades(toyPrepSim(objs, times = list(start = 2001, end = 2001), params = p), debug = FALSE) + first <- list(ig = data.table::copy(sim$fireSense_igAndEscapePred_Covariates), + sp = data.table::copy(sim$fireSense_SpreadCovariates)) + sim <- change(sim) + SpaDES.core::end(sim) <- 2001.75 + n <- countFuelClassCalls(sim <- SpaDES.core::spades(sim, debug = FALSE)) + list(n = n, first = first, after = list(ig = sim$fireSense_igAndEscapePred_Covariates, + sp = sim$fireSense_SpreadCovariates)) +} + +test_that("ignition and spread prepared together build once per year, as in a hindcast", { + n <- countFuelClassCalls(reps <- toyPrepRun(toyObjects(), times = repTimes, + params = list(fireTimeStep = 0.25))) + once <- toyPrepRun(toyObjects(), times = list(start = 2001, end = 2001)) + expect_identical(n, 1L) # one year, both events, was 8 events' worth of builds (2 per step, shared) + expect_identical(covDF(reps$fireSense_SpreadCovariates), covDF(once$fireSense_SpreadCovariates)) + expect_identical(covDF(reps$fireSense_igAndEscapePred_Covariates), covDF(once$fireSense_igAndEscapePred_Covariates)) +}) + +test_that("a change that keeps the sum (two pixels swapped) rebuilds", { + r <- stepsAfterChange(toyObjects(), function(sim) { + v <- terra::values(sim$pixelGroupMap, mat = FALSE) + v[c(1, 3)] <- v[c(3, 1)] + sim$pixelGroupMap <- terra::setValues(sim$pixelGroupMap, v) + sim + }, "fireSense_spreadPredict") + expect_identical(r$n, 1L) + expect_false(isTRUE(all.equal(covDF(r$first$sp), covDF(r$after$sp)))) +}) + +test_that("a land-cover change between steps of one year rebuilds, with several ELFs too", { + o <- toyObjects() + o$landcoverDT <- NULL + o$sppEquivs <- list( + data.table::data.table(LandR = c("Pice_mar", "Pinu_ban", "Popu_tre"), FuelClass = c("class1", "class1", "class2")), + data.table::data.table(LandR = c("Pice_mar", "Pinu_ban", "Popu_tre"), FuelClass = c("Pice_mar", "Pinu_ban", "decid"))) + o$nonForestedLCCGroupsList <- list(list(nfLCC_wetland = 19L, nfLCC_grass = 16L), list(nfLCC_wetgrs = c(16L, 19L))) + o$missingLCCgroupList <- list("nfLCC_grass", "nfLCC_wetgrs") + r <- stepsAfterChange(o, function(sim) { # a forest pixel and a wetland pixel swap land cover + v <- terra::values(sim$rstLCC_RTM, mat = FALSE) + v[c(1, 9)] <- v[c(9, 1)] + sim$rstLCC_RTM <- terra::setValues(sim$rstLCC_RTM, v) + sim + }, "fireSense_spreadPredict") + expect_identical(r$n, 2L) # the rebuild, one per ELF + expect_false(isTRUE(all.equal(covDF(r$first$sp), covDF(r$after$sp)))) +}) + +test_that("a change of the simulated ELF rebuilds", { + r <- stepsAfterChange(toyObjects(), function(sim) { sim$.ELFind <- "7"; sim }, "fireSense_spreadPredict") + expect_identical(r$n, 1L) +}) From 7d4b997105e2279aa1b4fd08345fccdfe87992bf Mon Sep 17 00:00:00 2001 From: Eliot McIntire Date: Sun, 15 Nov 2026 00:27:37 -0700 Subject: [PATCH 4/5] Replay through SpaDES.core outputs()/inputs(); no createsOutput of other modules' objects; sf and tables via qs2 Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014G18Mik3fH4z5M5CyMSUS6 --- NEWS.md | 2 +- fireSense_dataPrepPredict.R | 154 +++++++++++++++++++-------------- tests/testthat/test-init.R | 7 +- tests/testthat/test-metadata.R | 6 -- tests/testthat/test-replay.R | 109 ++++++++++++++++++----- 5 files changed, 176 insertions(+), 102 deletions(-) diff --git a/NEWS.md b/NEWS.md index de6bd6d..9e81642 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ # fireSense_dataPrepPredict (development version) -* New parameters `replaySave` (default `FALSE`) and `replayFrom` (default `NA`): no change unless set. `replaySave = TRUE` writes, once after the first covariate build, the spread-prediction inputs that do not depend on the spread parameters to `outputPath(sim)`: `fireSense_SpreadCovariates`, `flammableRTM`, `rasterToMatch`, `rasterToMatchELF`, `rasterToMatchLargeELF`, `studyAreaWithSpreadParams`, `nonEscapedFireSizesHa`, `ignitionFirePoints` and `firePolys` (those present): `replay_.tif` and `replay_.gpkg` for terra objects (`terra::writeRaster()`, `terra::writeVector()`), the rest in `replay_spreadInputs.qs2`. `replayFrom = ` loads them in `.inputObjects` and schedules no covariate event, so a job needs no climate, vegetation, fit or ELF module. `qs2` is now in `reqdPkgs`. +* New parameters `replaySave` (default `FALSE`) and `replayFrom` (default `NA`): no change unless set. `replaySave = TRUE` writes, once after the first covariate build, the spread-prediction inputs that do not depend on the spread parameters to `outputPath(sim)` through SpaDES.core's `outputs()` (its `save` event): `fireSense_SpreadCovariates`, `flammableRTM`, `rasterToMatch`, `rasterToMatchELF`, `rasterToMatchLargeELF`, `studyAreaWithSpreadParams`, `nonEscapedFireSizesHa`, `ignitionFirePoints` and `firePolys` (those present), as `replay__year