From bdd7fbf714809c092cbd20a0627756e18466c501 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 18:41:39 -0600 Subject: [PATCH 1/2] GH-55478: [R] Vendor thirdparty sources needed for Parquet, codecs, compute and JSON Ship trimmed source tarballs of the thirdparty dependencies needed for Parquet, the compression codecs, RE2/utf8proc, JSON and mimalloc in the R package so that offline source builds (e.g. on CRAN) can include those features instead of falling back to a minimal build. * tools/vendor_dependencies.sh (run by `make sync-cpp`) downloads, trims and vendors the tarballs into tools/vendored, and records each one's upstream and trimmed checksums in tools/vendored/checksums.txt. A dependency can provide its own trim script at tools/trim/.sh. * When building the package's copy of the C++ source (tools/cpp), nixlibs.R uses the vendored tarballs that were trimmed from the versions that versions.txt pins, replacing their checksums in versions.txt with the trimmed ones. When offline, it only turns off the features whose dependencies aren't vendored. * The offline CI job (now test-r-offline-vendored) checks that the expected features are available after the build. * Add inst/COPYRIGHTS listing the vendored dependencies' licenses. Co-Authored-By: Claude Opus 5.5 --- ci/scripts/r_test.sh | 20 +++ dev/release/rat_exclude_files.txt | 1 + dev/tasks/tasks.yml | 30 +++- r/.Rbuildignore | 1 + r/.gitignore | 2 + r/DESCRIPTION | 1 + r/inst/COPYRIGHTS | 78 ++++++++++ r/tools/nixlibs.R | 217 ++++++++++++++++++++++---- r/tools/test-nixlibs.R | 227 ++++++++++++++++++++++----- r/tools/vendor_dependencies.sh | 248 +++++++++++++++++++++++++++--- 10 files changed, 727 insertions(+), 98 deletions(-) create mode 100644 r/inst/COPYRIGHTS diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 20e6062e847f..ac9fbf2c479d 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -158,6 +158,26 @@ if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then cp /etc/resolv.conf{.bak,} fi +if [ -n "${ARROW_R_EXPECTED_FEATURES:-}" ]; then + # The tests skip what the build doesn't support, so make sure the features + # we expect (e.g. from the vendored thirdparty sources) were actually built. + # ARROW_R_EXPECTED_FEATURES is a comma-separated list of + # names(arrow::arrow_info()$capabilities) and memory pool backends + # (e.g. mimalloc). + ${R_BIN} -s -e " + library(arrow, lib.loc = 'check/arrow.Rcheck') + info <- arrow_info() + print(info\$capabilities) + print(info\$memory_pool\$available_backends) + available <- c(names(which(info\$capabilities)), info\$memory_pool\$available_backends) + expected <- strsplit(Sys.getenv('ARROW_R_EXPECTED_FEATURES'), ',')[[1]] + missing <- setdiff(expected, available) + if (length(missing) > 0) { + stop('Expected features are not available: ', paste(missing, collapse = ', ')) + } + " +fi + AFTER=$(ls -alh ~/) if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then # Ignore ~/.TinyTex/ and ~/R/ because it has many files. diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 3c5518a9596e..ef6e1a4f88dd 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -97,6 +97,7 @@ r/tests/testthat/_snaps/*.md r/inst/include/cpp11.hpp r/inst/include/cpp11/*.hpp r/tools/nixlibs-allowlist.txt +r/inst/COPYRIGHTS .gitattributes ruby/red-arrow/.yardopts ruby/red-arrow-format/lib/arrow-format/org/* diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d9351d3afe0a..8ea23b6d93b4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -633,14 +633,30 @@ tasks: template: r/github.linux.offline.build.yml # Simulate a CRAN builder without network access: the build should detect - # that it is offline and fall back to a minimal build without downloading - # anything. - test-r-offline-minimal: - ci: github - template: docker-tests/github.linux.yml - params: - flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true' + # that it is offline and build libarrow without downloading anything, using + # the thirdparty sources vendored in the package (tools/vendored) to still + # include Parquet, Dataset, JSON, the compression codecs, etc. + # ARROW_SOURCE_HOME='' makes it use the C++ source copied into tools/cpp + # (like the CRAN package does), whose versions.txt has the checksums of the + # trimmed tarballs. The codecs that are off by default on Linux are turned on + # so that every vendored tarball is built. + test-r-offline-vendored: + ci: github + template: docker-tests/github.linux.yml + params: + flags: >- + -e ARROW_R_TEST_OFFLINE=true + -e NOT_CRAN=false + -e SKIP_VIGNETTES=true + -e ARROW_SOURCE_HOME='' + -e ARROW_WITH_BROTLI=ON + -e ARROW_WITH_BZ2=ON + -e ARROW_WITH_ZSTD=ON + -e ARROW_R_EXPECTED_FEATURES=dataset,parquet,json,mimalloc,re2,utf8proc,snappy,gzip,brotli,zstd,lz4,bz2 image: r + # A full libarrow build with 2 cores (NOT_CRAN=false) and no sccache + # (it's offline) + timeout: 180 # This one has -flto=auto test-r-ubuntu-22.04: diff --git a/r/.Rbuildignore b/r/.Rbuildignore index deaf5dd3948c..698ccf3f7afc 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -32,3 +32,4 @@ STYLE.md ^inst/__pycache__$ ^bootstrap.R$ air.toml +^tools/vendored-downloads$ diff --git a/r/.gitignore b/r/.gitignore index a1295ea4e0b5..e16bc75f459b 100644 --- a/r/.gitignore +++ b/r/.gitignore @@ -35,3 +35,5 @@ extra-tests/files # Source tarballs of required dependencies that we ship for offline builds. # They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh). /tools/vendored/ +# Cache of the upstream (untrimmed) tarballs used by tools/vendor_dependencies.sh +/tools/vendored-downloads/ diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 1e5733c37cd5..19918beab685 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -24,6 +24,7 @@ Description: 'Apache' 'Arrow' is a cross-language package provides an interface to the 'Arrow C++' library. Depends: R (>= 4.2) License: Apache License (>= 2.0) +Copyright: See file inst/COPYRIGHTS URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/ BugReports: https://github.com/apache/arrow/issues Encoding: UTF-8 diff --git a/r/inst/COPYRIGHTS b/r/inst/COPYRIGHTS new file mode 100644 index 000000000000..779b8b2a5fa8 --- /dev/null +++ b/r/inst/COPYRIGHTS @@ -0,0 +1,78 @@ +The arrow R package is Copyright The Apache Software Foundation and is licensed +under the Apache License, Version 2.0. The Arrow C++ library source copied into +tools/cpp includes code from other projects; see tools/LICENSE.txt and +tools/NOTICE.txt in the source package. + +To build the Arrow C++ library without network access, the source package also +includes (trimmed) source tarballs of these thirdparty dependencies in +tools/vendored. They are only used when the dependency isn't found on the +system, and are statically linked into the package when used. Each tarball +contains the dependency's full license text. + +Boost (headers used by Apache Thrift) + Copyright the Boost authors + Boost Software License 1.0 + +Brotli + Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors + MIT License + +bzip2 + Copyright (C) 1996-2019 Julian R Seward + BSD-style license (see LICENSE in the tarball) + +LZ4 (lib/ only) + Copyright (c) 2011-2020, Yann Collet + BSD 2-Clause License + +mimalloc + Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen + MIT License + +RapidJSON + Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip + MIT License (with code under the BSD and JSON licenses, see license.txt in + the tarball) + +RE2 + Copyright (c) 2009 The RE2 Authors + BSD 3-Clause License + +simdjson + Copyright 2018-2025 The simdjson authors + Apache License 2.0 or MIT License + +Snappy + Copyright 2011, Google Inc. + BSD 3-Clause License + +Apache Thrift + Copyright (C) 2006-2019, The Apache Software Foundation + Apache License 2.0 + +uriparser (library sources only) + Copyright (C) 2007, Weijia Song and Sebastian Pipping + BSD 3-Clause License + +utf8proc + Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Tony Kelman, + Jonas Fonseca, and other contributors + Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany + MIT License, with Unicode data under the Unicode Data Files and Software + License + +xsimd + Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and + Martin Renou + Copyright (c) 2016, QuantStack + Copyright (c) 2018, Serge Guelton + BSD 3-Clause License + +zlib + Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler + zlib License + +Zstandard (zstd) + Copyright (c) Meta Platforms, Inc. and affiliates + BSD 3-Clause License (zstd is dual-licensed BSD/GPLv2; it is used under the + BSD license) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 226894372ec5..a2c321647bd5 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -486,7 +486,7 @@ find_local_source <- function() { # The second probably occurs if we're installing the arrow R package cpp_dir_options <- c( file.path(Sys.getenv("ARROW_SOURCE_HOME", ".."), "cpp"), - "tools/cpp" + package_cpp_dir ) for (cpp_dir in cpp_dir_options) { if (file.exists(file.path(cpp_dir, "src/arrow/api.h"))) { @@ -625,30 +625,35 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- with_cloud_support(env_var_list) env_var_list <- with_wasm_support(env_var_list) - # turn_off_all_optional_features() needs to happen after - # with_cloud_support(), since it might turn features ON. + # turn_off_all_optional_features() and turn_off_unvendored_features() need to + # happen after with_cloud_support(), since it might turn features ON. thirdparty_deps_unavailable <- !download_ok && !dir.exists(thirdparty_dependency_dir) && !env_is("ARROW_DEPENDENCY_SOURCE", "system") do_minimal_build <- env_is("LIBARROW_MINIMAL", "true") + # Thirdparty source tarballs shipped in the package + vendored <- usable_vendored_dependencies(src_dir) if (do_minimal_build) { env_var_list <- turn_off_all_optional_features(env_var_list) } else if (thirdparty_deps_unavailable) { cat(paste0( "*** Building C++ library from source, but downloading thirdparty dependencies\n", - " is not possible, so this build will turn off all thirdparty features.\n", + " is not possible, so this build will turn off the thirdparty features\n", + " whose sources are not included in the package.\n", " See installation guide for details:\n", " https://arrow.apache.org/docs/r/articles/install.html\n" )) - env_var_list <- turn_off_all_optional_features(env_var_list) + env_var_list <- turn_off_unvendored_features(env_var_list, names(vendored)) } else if (dir.exists(thirdparty_dependency_dir)) { # Add the *_SOURCE_URL env vars env_var_list <- set_thirdparty_urls(env_var_list) } - # Some thirdparty dependencies are required even for a minimal build, so they - # can't be turned off like the ones above. - env_var_list <- with_vendored_dependencies(env_var_list) + # Point the build at the vendored sources of the thirdparty dependencies. + # Some of them are required even for a minimal build, so they can't be turned + # off like the ones above. + env_var_list <- with_vendored_dependencies(env_var_list, vendored) + use_vendored_checksums(src_dir, env_var_list, vendored) env_vars <- env_vars_as_string(env_var_list) lg("arrow %s", ifelse(quietly, "", paste("with", env_vars)), .indent = "****") @@ -795,7 +800,9 @@ cmake_version <- function(cmd = "cmake") { ) } -turn_off_all_optional_features <- function(env_var_list) { +# `keep` is a vector of features (names in `turn_off` below, or +# "ARROW_SIMD_LEVEL") to leave as they are. +turn_off_all_optional_features <- function(env_var_list, keep = character()) { # Because these are done as environment variables (as opposed to build flags), # setting these to "OFF" overrides any previous setting. We don't need to # check the existing value. @@ -822,19 +829,53 @@ turn_off_all_optional_features <- function(env_var_list) { "ARROW_WITH_ZLIB" = "OFF", "ARROW_WITH_ZSTD" = "OFF", "ARROW_WITH_RE2" = "OFF", - "ARROW_WITH_UTF8PROC" = "OFF", + "ARROW_WITH_UTF8PROC" = "OFF" + ) + turn_off <- turn_off[setdiff(names(turn_off), keep)] + if (!("ARROW_SIMD_LEVEL" %in% keep)) { # The syntax to turn off XSIMD is different. # Pull existing value of EXTRA_CMAKE_FLAGS first (must be defined) - "EXTRA_CMAKE_FLAGS" = paste( - env_var_list[["EXTRA_CMAKE_FLAGS"]], - "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE" + turn_off <- c( + turn_off, + "EXTRA_CMAKE_FLAGS" = paste( + env_var_list[["EXTRA_CMAKE_FLAGS"]], + "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE" + ) ) - ) + } # Create a new env_var_list, with the values of turn_off set. # replace() also adds new values if they didn't exist before replace(env_var_list, names(turn_off), turn_off) } +# Without network access, libarrow can only build the thirdparty dependencies +# whose source tarballs are shipped in the package (see +# `with_vendored_dependencies()`), so turn off every optional feature except +# those whose dependencies are all vendored. Features that are kept are left as +# they are (e.g. as the user or the platform defaults set them), not turned on. +# `available` is the ARROW_*_URL names of the usable vendored tarballs (see +# `usable_vendored_dependencies()`). +turn_off_unvendored_features <- function(env_var_list, available) { + has_all <- vapply(vendored_features, function(deps) all(deps %in% available), logical(1)) + keep <- names(vendored_features)[has_all] + if ("ARROW_PARQUET" %in% keep) { + # Dataset needs no thirdparty dependencies itself, but it's only useful + # with Parquet (and we've always turned them on and off together) + keep <- c(keep, "ARROW_DATASET") + } + if ("ARROW_XSIMD_URL" %in% available) { + keep <- c(keep, "ARROW_SIMD_LEVEL") + } + if (!quietly) { + lg( + "Features with vendored dependencies: %s", + if (length(keep)) paste(keep, collapse = ", ") else "none", + .indent = "****" + ) + } + turn_off_all_optional_features(env_var_list, keep = keep) +} + get_component_names <- function() { if (!isTRUE(Sys.which("bash") != "")) { stop("nixlibs.R requires bash to be installed and available in your PATH") @@ -874,26 +915,53 @@ set_thirdparty_urls <- function(env_var_list) { env_var_list } -# libarrow requires some thirdparty dependencies, even for a minimal build. -# The libarrow build uses them from the system if they are available and -# otherwise builds them from source, downloading from ARROW_*_URL. To make sure -# that works without network access, `make sync-cpp` saves their source -# tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here. +# The libarrow build uses thirdparty dependencies from the system if they are +# available and otherwise builds them from source, downloading from ARROW_*_URL. +# Some are required even for a minimal build. To make sure that works without +# network access, `make sync-cpp` saves (trimmed) source tarballs into +# vendored_dependency_dir and we point ARROW_*_URL at them here. # This does not change whether the system versions are used. -with_vendored_dependencies <- function(env_var_list) { - if (!dir.exists(vendored_dependency_dir)) { +# `vendored` is the result of `usable_vendored_dependencies()`. +with_vendored_dependencies <- function(env_var_list, vendored) { + for (env_varname in names(vendored)) { + if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { + # Respect the user's (or set_thirdparty_urls()'s) choice + next + } + env_var_list <- replace(env_var_list, env_varname, vendored[[env_varname]]) + } + env_var_list +} + +# Returns the paths of the vendored tarballs that the libarrow source in +# `src_dir` can use, named by their ARROW_*_URL. +# The tarballs are trimmed, so the libarrow build only accepts them once +# `use_vendored_checksums()` has put their checksums in versions.txt. We only +# do that to the package's own copy of the source (tools/cpp), not e.g. ../cpp +# of a git checkout, and only for tarballs that were trimmed from the version +# that versions.txt pins (according to the checksums.txt file that +# tools/vendor_dependencies.sh writes). +usable_vendored_dependencies <- function(src_dir) { + manifest <- read_vendored_checksums() + if (is.null(manifest)) { # e.g. building from a git checkout without `make sync-cpp` if (!quietly) { lg("No vendored dependencies found, they will be built with CMake if needed", .indent = "****") } - return(env_var_list) + return(character()) } + is_package_cpp_dir <- !is.null(src_dir) && + identical(normalizePath(src_dir, mustWork = FALSE), normalizePath(package_cpp_dir, mustWork = FALSE)) + if (!is_package_cpp_dir) { + if (!quietly) { + lg("Not using vendored dependencies, they are only used to build %s", package_cpp_dir, .indent = "****") + } + return(character()) + } + expected <- read_thirdparty_checksums(file.path(src_dir, "thirdparty", "versions.txt")) deps_df <- get_component_names() + usable <- character() for (env_varname in vendored_dependencies) { - if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { - # Respect the user's (or set_thirdparty_urls()'s) choice - next - } filename <- deps_df$filename[deps_df$env_varname == env_varname] tarball <- file.path(vendored_dependency_dir, filename) if (length(tarball) != 1 || !file.exists(tarball)) { @@ -902,9 +970,68 @@ with_vendored_dependencies <- function(env_var_list) { } next } - env_var_list <- replace(env_var_list, env_varname, normalizePath(tarball)) + checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) + entry <- manifest[manifest$filename == filename, , drop = FALSE] + # versions.txt has the vendored checksum if use_vendored_checksums() has + # already run on it (e.g. when installing from the r/ directory again) + matches <- nrow(entry) == 1 && + unname(expected[checksum_varname]) %in% c(entry$upstream_sha256, entry$vendored_sha256) + if (!matches) { + if (!quietly) { + lg( + "Vendored source for %s doesn't match %s, it will be downloaded if needed", + env_varname, + file.path(src_dir, "thirdparty", "versions.txt"), + .indent = "****" + ) + } + next + } + usable[[env_varname]] <- normalizePath(tarball) } - env_var_list + usable +} + +# Read the checksums.txt that tools/vendor_dependencies.sh writes, with the +# checksums of each vendored tarball before (upstream) and after trimming +read_vendored_checksums <- function() { + manifest_file <- file.path(vendored_dependency_dir, "checksums.txt") + if (!file.exists(manifest_file)) { + return(NULL) + } + read.table(manifest_file, header = TRUE, stringsAsFactors = FALSE) +} + +# The libarrow build checks the checksums of the thirdparty tarballs against +# src_dir's versions.txt, so replace the upstream checksums of the vendored +# tarballs that `env_var_list` points at with the trimmed tarballs' checksums. +use_vendored_checksums <- function(src_dir, env_var_list, vendored) { + used <- vendored[vapply( + names(vendored), + function(env_varname) identical(env_var_list[[env_varname]], vendored[[env_varname]]), + logical(1) + )] + if (length(used) == 0) { + return(invisible()) + } + manifest <- read_vendored_checksums() + versions_txt <- file.path(src_dir, "thirdparty", "versions.txt") + lines <- readLines(versions_txt) + for (env_varname in names(used)) { + checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) + checksum <- manifest$vendored_sha256[manifest$filename == basename(used[[env_varname]])] + lines <- sub(paste0("^", checksum_varname, "=.*$"), paste0(checksum_varname, "=", checksum), lines) + } + writeLines(lines, versions_txt) +} + +# Parse the *_BUILD_SHA256_CHECKSUM=... lines of a versions.txt into a named vector +read_thirdparty_checksums <- function(versions_txt) { + if (!file.exists(versions_txt)) { + return(character()) + } + lines <- grep("^[A-Z0-9_]+_BUILD_SHA256_CHECKSUM=", readLines(versions_txt), value = TRUE) + setNames(sub("^[^=]*=", "", lines), sub("=.*$", "", lines)) } # this is generally about features that people asked for via environment variables, but @@ -1140,11 +1267,37 @@ if (download_libarrow_ok && !is_release && !test_mode) { # `create_package_with_all_dependencies()` in install-arrow.R thirdparty_dependency_dir <- Sys.getenv("ARROW_THIRDPARTY_DEPENDENCY_DIR", "tools/thirdparty_dependencies") -# Source tarballs of dependencies that libarrow requires even for a minimal build. -# These are always shipped in the package (see `with_vendored_dependencies()`). -# If you change this path or list, you also need to edit tools/vendor_dependencies.sh +# Trimmed source tarballs of thirdparty dependencies, shipped in the package so +# that building without network access still works and can include the features +# below (see `with_vendored_dependencies()` and `turn_off_unvendored_features()`). +# If you change this path or these lists, you also need to edit +# tools/vendor_dependencies.sh vendored_dependency_dir <- "tools/vendored" -vendored_dependencies <- c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL") +# The copy of the libarrow source in the package (see `make sync-cpp`) +package_cpp_dir <- "tools/cpp" +# Optional features and the ARROW_*_URL of each dependency they need (when the +# dependency is built from source). See "Some EP's require other EP's" in +# cpp/cmake_modules/ThirdpartyToolchain.cmake +vendored_features <- list( + # Parquet turns on Thrift and simdjson; Thrift needs zlib and Boost + ARROW_PARQUET = c("ARROW_BOOST_URL", "ARROW_SIMDJSON_URL", "ARROW_THRIFT_URL", "ARROW_ZLIB_URL"), + ARROW_JSON = c("ARROW_RAPIDJSON_URL", "ARROW_SIMDJSON_URL"), + ARROW_MIMALLOC = "ARROW_MIMALLOC_URL", + ARROW_WITH_BROTLI = "ARROW_BROTLI_URL", + ARROW_WITH_BZ2 = "ARROW_BZIP2_URL", + ARROW_WITH_LZ4 = "ARROW_LZ4_URL", + ARROW_WITH_RE2 = "ARROW_RE2_URL", + ARROW_WITH_SNAPPY = "ARROW_SNAPPY_URL", + ARROW_WITH_UTF8PROC = "ARROW_UTF8PROC_URL", + ARROW_WITH_ZLIB = "ARROW_ZLIB_URL", + ARROW_WITH_ZSTD = "ARROW_ZSTD_URL" +) +# uriparser and xsimd are required even for a minimal build +vendored_dependencies <- unique(c( + "ARROW_URIPARSER_URL", + "ARROW_XSIMD_URL", + unlist(vendored_features, use.names = FALSE) +)) arrow_versioned <- paste0("arrow-", VERSION) # configure.win uses a different libarrow dir and the zip is already nested diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index 9d8abccfbb5e..0320658d41d4 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -233,58 +233,213 @@ test_that("find_latest_nightly()", { ) }) -test_that("with_vendored_dependencies()", { - vendored_dir <- tempfile() - dir.create(vendored_dir) - on.exit(unlink(vendored_dir, recursive = TRUE), add = TRUE) - original_dir <- nixlibs_env$vendored_dependency_dir - original_get_component_names <- nixlibs_env$get_component_names - original_quietly <- nixlibs_env$quietly - on.exit(nixlibs_env$vendored_dependency_dir <- original_dir, add = TRUE) - on.exit(nixlibs_env$get_component_names <- original_get_component_names, add = TRUE) - on.exit(nixlibs_env$quietly <- original_quietly, add = TRUE) - nixlibs_env$vendored_dependency_dir <- vendored_dir - nixlibs_env$get_component_names <- function() { - data.frame( - env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL"), - filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz") +# Set up a fake vendored_dependency_dir and package_cpp_dir for the vendored +# dependency tests. `vendored` is a named vector of filename = upstream +# checksum in checksums.txt (the trimmed checksum is "-trimmed"), +# `expected` is the checksums in the source dir's versions.txt, named by +# ARROW_*_URL. +local_vendored_dependencies <- function(vendored, expected, env = parent.frame()) { + vendored_dir <- withr::local_tempdir(.local_envir = env) + src_dir <- withr::local_tempdir(.local_envir = env) + for (filename in names(vendored)) { + file.create(file.path(vendored_dir, filename)) + } + if (length(vendored)) { + writeLines( + c( + "filename upstream_sha256 vendored_sha256", + paste(names(vendored), vendored, paste0(vendored, "-trimmed")) + ), + file.path(vendored_dir, "checksums.txt") ) } - withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) + dir.create(file.path(src_dir, "thirdparty")) + writeLines( + c( + "# a comment", + paste0(sub("_URL$", "_BUILD_SHA256_CHECKSUM", names(expected)), "=", expected) + ), + file.path(src_dir, "thirdparty", "versions.txt") + ) + local_binding("vendored_dependency_dir", vendored_dir, env) + local_binding("package_cpp_dir", src_dir, env) + local_binding( + "get_component_names", + function() { + data.frame( + env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL", "ARROW_JEMALLOC_URL"), + filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz", "jemalloc-5.3.0.tar.bz2") + ) + }, + env + ) + list(vendored_dir = vendored_dir, src_dir = src_dir) +} + +# Temporarily replace a variable in nixlibs_env +local_binding <- function(name, value, env = parent.frame()) { + original <- nixlibs_env[[name]] + assign(name, value, envir = nixlibs_env) + withr::defer(assign(name, original, envir = nixlibs_env), envir = env) +} - # Without the vendored tarballs, nothing is set - nixlibs_env$quietly <- TRUE - expect_silent( - expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")) +test_that("usable_vendored_dependencies()", { + local_binding("quietly", TRUE) + # Nothing vendored + dirs <- local_vendored_dependencies(character(), c(ARROW_URIPARSER_URL = "aaa")) + expect_identical(usable_vendored_dependencies(dirs$src_dir), character()) + + dirs <- local_vendored_dependencies( + c( + "uriparser-1.0.2.tar.bz2" = "aaa", + "xsimd-14.3.0.tar.gz" = "bbb", + "zstd-1.5.7.tar.gz" = "ccc", + # Not in vendored_dependencies, so it isn't used + "jemalloc-5.3.0.tar.bz2" = "ddd" + ), + c( + ARROW_URIPARSER_URL = "aaa", + # A different version than the vendored one + ARROW_XSIMD_URL = "other", + # Already has the trimmed checksum (use_vendored_checksums() has run) + ARROW_ZSTD_URL = "ccc-trimmed", + ARROW_JEMALLOC_URL = "ddd" + ) + ) + expect_identical( + usable_vendored_dependencies(dirs$src_dir), + c( + ARROW_URIPARSER_URL = normalizePath(file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2")), + ARROW_ZSTD_URL = normalizePath(file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz")) + ) + ) + # Mismatches are only reported with verbose logging + local_binding("quietly", FALSE) + expect_output( + usable_vendored_dependencies(dirs$src_dir), + "Vendored source for ARROW_XSIMD_URL doesn't match" + ) + + # Only used to build package_cpp_dir (not e.g. ../cpp of a git checkout) + other_src_dir <- withr::local_tempdir() + file.copy(file.path(dirs$src_dir, "thirdparty"), other_src_dir, recursive = TRUE) + expect_output( + expect_identical(usable_vendored_dependencies(other_src_dir), character()), + "Not using vendored dependencies" + ) + expect_output( + expect_identical(usable_vendored_dependencies(NULL), character()), + "Not using vendored dependencies" ) - # The missing tarballs are only reported with verbose logging - nixlibs_env$quietly <- FALSE + + # A missing tarball isn't used, even if it's in checksums.txt + unlink(file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz")) expect_output( - expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")), - "Vendored source for ARROW_URIPARSER_URL not found" + expect_named(usable_vendored_dependencies(dirs$src_dir), "ARROW_URIPARSER_URL"), + "Vendored source for ARROW_ZSTD_URL not found" ) - nixlibs_env$quietly <- TRUE +}) - uriparser <- file.path(vendored_dir, "uriparser-1.0.2.tar.bz2") - xsimd <- file.path(vendored_dir, "xsimd-14.3.0.tar.gz") - # Only dependencies in vendored_dependencies are used - zstd <- file.path(vendored_dir, "zstd-1.5.7.tar.gz") - file.create(uriparser, xsimd, zstd) +test_that("use_vendored_checksums()", { + local_binding("quietly", TRUE) + dirs <- local_vendored_dependencies( + c("uriparser-1.0.2.tar.bz2" = "aaa", "xsimd-14.3.0.tar.gz" = "bbb", "zstd-1.5.7.tar.gz" = "ccc"), + c(ARROW_URIPARSER_URL = "aaa", ARROW_XSIMD_URL = "bbb", ARROW_ZSTD_URL = "ccc", ARROW_JEMALLOC_URL = "ddd") + ) + versions_txt <- file.path(dirs$src_dir, "thirdparty", "versions.txt") + original <- readLines(versions_txt) + vendored <- c( + ARROW_URIPARSER_URL = file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2"), + ARROW_XSIMD_URL = file.path(dirs$vendored_dir, "xsimd-14.3.0.tar.gz"), + ARROW_ZSTD_URL = file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz") + ) + + # Nothing is pointed at the vendored tarballs + use_vendored_checksums(dirs$src_dir, list(), vendored) + expect_identical(readLines(versions_txt), original) + + # Only the ones that are (e.g. not when the user set ARROW_XSIMD_URL) + env_var_list <- list( + ARROW_URIPARSER_URL = vendored[["ARROW_URIPARSER_URL"]], + ARROW_XSIMD_URL = "https://example.com/xsimd.tar.gz", + ARROW_ZSTD_URL = vendored[["ARROW_ZSTD_URL"]] + ) + use_vendored_checksums(dirs$src_dir, env_var_list, vendored) expect_identical( - with_vendored_dependencies(list(A = "1")), - list(A = "1", ARROW_URIPARSER_URL = normalizePath(uriparser), ARROW_XSIMD_URL = normalizePath(xsimd)) + readLines(versions_txt), + c( + "# a comment", + "ARROW_URIPARSER_BUILD_SHA256_CHECKSUM=aaa-trimmed", + "ARROW_XSIMD_BUILD_SHA256_CHECKSUM=bbb", + "ARROW_ZSTD_BUILD_SHA256_CHECKSUM=ccc-trimmed", + "ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=ddd" + ) + ) + # and they are still usable afterwards + expect_named( + usable_vendored_dependencies(dirs$src_dir), + c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL") + ) +}) + +test_that("with_vendored_dependencies()", { + vendored <- c(ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") + withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) + + expect_identical(with_vendored_dependencies(list(A = "1"), character()), list(A = "1")) + expect_identical( + with_vendored_dependencies(list(A = "1"), vendored), + list(A = "1", ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") ) # Already set (e.g. by set_thirdparty_urls()) is respected expect_identical( - with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2")), - list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = normalizePath(xsimd)) + with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2"), vendored), + list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") ) # As is a user-set env var withr::local_envvar(c(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2")) expect_identical( - with_vendored_dependencies(list(A = "1")), - list(A = "1", ARROW_XSIMD_URL = normalizePath(xsimd)) + with_vendored_dependencies(list(A = "1"), vendored), + list(A = "1", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") + ) +}) + +test_that("turn_off_unvendored_features()", { + local_binding("quietly", TRUE) + env_var_list <- list(EXTRA_CMAKE_FLAGS = "", ARROW_WITH_ZSTD = "ON", ARROW_S3 = "ON") + + # Nothing vendored: the same as turn_off_all_optional_features() + expect_identical( + turn_off_unvendored_features(env_var_list, character()), + turn_off_all_optional_features(env_var_list) + ) + + # Everything vendored: only features that need unvendored dependencies are + # turned off, and the rest are left as they were (not turned on) + result <- turn_off_unvendored_features(env_var_list, vendored_dependencies) + expect_identical(result$ARROW_S3, "OFF") + expect_identical(result$ARROW_JEMALLOC, "OFF") + expect_identical(result$ARROW_WITH_ZSTD, "ON") + for (feature in setdiff(c(names(vendored_features), "ARROW_DATASET"), "ARROW_WITH_ZSTD")) { + expect_null(result[[feature]]) + } + expect_identical(result$EXTRA_CMAKE_FLAGS, "") + + # Parquet (and Dataset) need all of their dependencies + result <- turn_off_unvendored_features( + env_var_list, + c("ARROW_XSIMD_URL", "ARROW_THRIFT_URL", "ARROW_ZSTD_URL") + ) + expect_identical(result$ARROW_PARQUET, "OFF") + expect_identical(result$ARROW_DATASET, "OFF") + expect_identical(result$ARROW_WITH_ZSTD, "ON") + expect_identical(result$ARROW_WITH_SNAPPY, "OFF") + # SIMD is only turned off if xsimd isn't vendored + expect_identical(result$EXTRA_CMAKE_FLAGS, "") + expect_match( + turn_off_unvendored_features(env_var_list, "ARROW_ZSTD_URL")$EXTRA_CMAKE_FLAGS, + "-DARROW_SIMD_LEVEL=NONE" ) }) diff --git a/r/tools/vendor_dependencies.sh b/r/tools/vendor_dependencies.sh index caa28bff5ce2..686ce981e286 100644 --- a/r/tools/vendor_dependencies.sh +++ b/r/tools/vendor_dependencies.sh @@ -17,30 +17,80 @@ # specific language governing permissions and limitations # under the License. -# libarrow requires some thirdparty dependencies, even for a minimal build. -# This script downloads the source tarballs that libarrow would otherwise -# download at build time, so that they can be shipped in the R package. -# nixlibs.R uses them when the dependencies are not found on the system, so -# that building without network access (e.g. on CRAN) still works. +# libarrow downloads the source of its thirdparty dependencies at build time +# when they aren't found on the system. This script downloads those source +# tarballs ahead of time so that they can be shipped in the R package, which +# lets building without network access (e.g. on CRAN) still include features +# like Parquet. nixlibs.R uses them when the dependencies are not found on the +# system, and turns off the features whose dependencies aren't vendored. +# +# To keep the package small, each tarball is trimmed down to what the libarrow +# build uses (no tests, docs, other language bindings, ...). Trimmed tarballs +# no longer match the checksums pinned in versions.txt, so this script records +# the upstream and trimmed checksums of each tarball in DESTDIR/checksums.txt. +# When building the package's copy of the libarrow source (tools/cpp), nixlibs.R +# uses the tarballs whose upstream checksum matches versions.txt and replaces +# it with the trimmed one. +# +# Trimming is done by, in order of preference: +# 1. tools/trim/.sh IN_TARBALL OUT_TARBALL, if it exists, where +# is the lowercase dependency name (e.g. ARROW_BOOST_URL -> boost). Use +# this for dependencies that need more than a few `rm`s. +# 2. Otherwise, trim_generic() plus the dependency-specific removals in +# trim_dependency() below. # # Usage: tools/vendor_dependencies.sh [DESTDIR] -# Must be run after tools/cpp has been synced (see `make sync-cpp`). +# Must be run after tools/cpp has been synced (see `make sync-cpp`). Needs curl +# and cmake (to handle .tar.bz2 without a bzip2 binary). +# +# Environment variables: +# ARROW_VENDOR_DOWNLOAD_DIR: where to cache the upstream (untrimmed) +# tarballs between runs (default: tools/vendored-downloads, which is +# ignored by git and R CMD build). set -euo pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DESTDIR="${1:-${SOURCE_DIR}/vendored}" +DOWNLOAD_DIR="${ARROW_VENDOR_DOWNLOAD_DIR:-${SOURCE_DIR}/vendored-downloads}" +VERSIONS_TXT="${SOURCE_DIR}/cpp/thirdparty/versions.txt" # If you change this list, you also need to edit -# `vendored_dependencies` in nixlibs.R +# `vendored_dependencies` (and `vendored_features`) in nixlibs.R VENDORED_DEPENDENCIES=( + # Required even for a minimal build ARROW_URIPARSER_URL ARROW_XSIMD_URL + # Parquet (ARROW_PARQUET turns on Thrift and simdjson; Thrift needs zlib and + # Boost headers) + ARROW_BOOST_URL + ARROW_SIMDJSON_URL + ARROW_THRIFT_URL + ARROW_ZLIB_URL + # Compression codecs + ARROW_BROTLI_URL + ARROW_BZIP2_URL + ARROW_LZ4_URL + ARROW_SNAPPY_URL + ARROW_ZSTD_URL + # Compute (RE2, utf8proc), JSON (RapidJSON, simdjson) and the allocator + ARROW_MIMALLOC_URL + ARROW_RAPIDJSON_URL + ARROW_RE2_URL + ARROW_UTF8PROC_URL ) +if [ ! -f "${VERSIONS_TXT}" ]; then + echo "${VERSIONS_TXT} not found. Run this from \`make sync-cpp\`." >&2 + exit 1 +fi + # Load `DEPENDENCIES` and the *_BUILD_SHA256_CHECKSUM variables. # shellcheck source=/dev/null -source "${SOURCE_DIR}/cpp/thirdparty/versions.txt" +source "${VERSIONS_TXT}" + +# Don't add macOS metadata (._* files, extended attributes) to the tarballs +export COPYFILE_DISABLE=1 sha256() { if command -v sha256sum > /dev/null 2>&1; then @@ -50,8 +100,150 @@ sha256() { fi } -mkdir -p "${DESTDIR}" -keep=() +# rm -rf the given paths, relative to the current directory +rmd() { + rm -rf "$@" +} + +# Remove everything in the current directory except the given names +keep_only() { + local entry name keep + for entry in * .[!.]*; do + [ -e "${entry}" ] || continue + keep=false + for name in "$@"; do + if [ "${entry}" = "${name}" ]; then + keep=true + fi + done + if [ "${keep}" = "false" ]; then + rm -rf "${entry}" + fi + done +} + +# Remove directories that no bundled build of our dependencies compiles. +# Arguments are paths (relative to the source root) to keep anyway. +trim_generic() { + local dir keep k + find . -depth -type d \( -name test -o -name tests -o -name testdata \ + -o -name doc -o -name docs -o -name example -o -name examples \ + -o -name benchmark -o -name benchmarks -o -name bench -o -name fuzz \) \ + -print | while read -r dir; do + keep=false + for k in "$@"; do + if [ "${dir}" = "./${k}" ]; then + keep=true + fi + done + if [ "${keep}" = "false" ]; then + rm -rf "${dir}" + fi + done +} + +# Trim an extracted source tree (the current directory) for the dependency +# named $1 (e.g. "thrift") +trim_dependency() { + case "$1" in + brotli) + rmd go python java csharp js research + # install() copies the man pages from docs/ + trim_generic docs + ;; + bzip2) + rm -f manual.* sample* ./*.html + trim_generic + ;; + lz4) + # Only lib/ is built (LZ4_BUILD_CLI=OFF). Files outside of lib/ are + # GPL-2.0-or-later, so don't ship the ones we don't need. + rmd programs contrib ossfuzz + trim_generic + (cd build && keep_only cmake) + ;; + mimalloc) + rmd ide + trim_generic + ;; + rapidjson) + rmd bin thirdparty + trim_generic + ;; + re2) + rmd benchlog python app + trim_generic + ;; + simdjson) + # Built from src/ and include/. singleheader/ is a 20 MB amalgamation. + rmd singleheader images jsonexamples tools scripts + trim_generic + ;; + thrift) + # Only the C++ library is built (BUILD_COMPILER=OFF, WITH_*=OFF) + rmd tutorial test compiler contrib doc debian + (cd lib && keep_only cpp CMakeLists.txt) + trim_generic + ;; + zlib) + rmd contrib + # zlib's CMakeLists.txt builds test/example.c unless ZLIB_BUILD_EXAMPLES=OFF + trim_generic test + ;; + zstd) + # ZSTD_BUILD_PROGRAMS=OFF + rmd contrib zlibWrapper programs + trim_generic + (cd build && keep_only cmake) + ;; + *) + trim_generic + ;; + esac +} + +# Extract $1, trim it for dependency $3 and write the result to $2 +trim_tarball() { + local in="$1" out="$2" name="$3" + local work src + if [ -f "${SOURCE_DIR}/trim/${name}.sh" ]; then + bash "${SOURCE_DIR}/trim/${name}.sh" "${in}" "${out}" + return + fi + work="$(mktemp -d)" + # Use CMake's tar (libarchive) for .tar.bz2: tar -j needs a bzip2 binary, + # which e.g. the R CI images don't have. + case "${in}" in + *.tar.bz2) (cd "${work}" && cmake -E tar xf "${in}") ;; + *) tar -xf "${in}" -C "${work}" ;; + esac + # Every tarball we vendor has a single top-level directory + src="$(cd "${work}" && ls)" + if [ "$(echo "${src}" | wc -l)" -ne 1 ] || [ ! -d "${work}/${src}" ]; then + echo "Expected a single top-level directory in ${in}" >&2 + rm -rf "${work}" + return 1 + fi + (cd "${work}/${src}" && trim_dependency "${name}") + case "${out}" in + *.tar.bz2) (cd "${work}" && cmake -E tar cjf "${out}" "${src}") ;; + *.tar.gz) tar -czf "${out}" -C "${work}" "${src}" ;; + *) + echo "Don't know how to compress ${out}" >&2 + rm -rf "${work}" + return 1 + ;; + esac + rm -rf "${work}" +} + +mkdir -p "${DESTDIR}" "${DOWNLOAD_DIR}" +# trim_tarball() changes directories, so it needs absolute paths +DOWNLOAD_DIR="$(cd "${DOWNLOAD_DIR}" && pwd)" +TRIM_DIR="$(mktemp -d)" +trap 'rm -rf "${TRIM_DIR}"' EXIT +keep=(checksums.txt) +sums="filename upstream_sha256 vendored_sha256"$'\n' for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do url="" for dep_packed in "${DEPENDENCIES[@]}"; do @@ -67,28 +259,38 @@ for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do echo "${vendored_url_var} not found in versions.txt" >&2 exit 1 fi - # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM + # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM and xsimd checksum_var="${vendored_url_var%_URL}_BUILD_SHA256_CHECKSUM" checksum="${!checksum_var}" + name="$(echo "${vendored_url_var}" | sed -e 's/^ARROW_//' -e 's/_URL$//' | tr '[:upper:]' '[:lower:]')" keep+=("${tar_name}") - out="${DESTDIR}/${tar_name}" - if [ -f "${out}" ] && [ "$(sha256 "${out}")" = "${checksum}" ]; then - continue + upstream="${DOWNLOAD_DIR}/${tar_name}" + if [ ! -f "${upstream}" ] || [ "$(sha256 "${upstream}")" != "${checksum}" ]; then + echo "Downloading ${url}" + curl --fail --silent --show-error --location --retry 3 --output "${upstream}" "${url}" + if [ "$(sha256 "${upstream}")" != "${checksum}" ]; then + echo "Checksum mismatch for ${upstream}" >&2 + rm -f "${upstream}" + exit 1 + fi fi - echo "Downloading ${url}" - curl --fail --silent --show-error --location --retry 3 --output "${out}" "${url}" - - if [ "$(sha256 "${out}")" != "${checksum}" ]; then - echo "Checksum mismatch for ${out}" >&2 - rm -f "${out}" - exit 1 - fi + # Trim into a temporary directory (keeping the file name, so the extension + # says how to compress it) so that a failure doesn't leave a partial file. + echo "Trimming ${tar_name}" + trim_tarball "${upstream}" "${TRIM_DIR}/${tar_name}" "${name}" + out="${DESTDIR}/${tar_name}" + mv "${TRIM_DIR}/${tar_name}" "${out}" + sums="${sums}${tar_name} ${checksum} $(sha256 "${out}")"$'\n' done -# Remove tarballs for other versions +# See read_vendored_checksums() in nixlibs.R +printf "%s" "${sums}" > "${DESTDIR}/checksums.txt" + +# Remove tarballs for other versions (or dependencies we no longer vendor) for file in "${DESTDIR}"/*; do + [ -e "${file}" ] || continue stale=true for name in "${keep[@]}"; do if [ "$(basename "${file}")" = "${name}" ]; then From dda15ba2cede1feaa08cbd8c7c260789b76fc2ed Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 18:42:01 -0600 Subject: [PATCH 2/2] GH-55478: [R] Trim the vendored Boost to the libraries Thrift needs The generic trim leaves a 29 MB Boost tarball. tools/trim/boost.sh keeps only the Boost libraries that build_boost() in ThirdpartyToolchain.cmake includes (minus ones the R build doesn't use, e.g. multiprecision) and their dependencies, as resolved by Boost's own CMake, and drops their tests, docs and examples. This brings the vendored Boost down to ~5 MB. Co-Authored-By: Claude Opus 5.5 --- r/.Rbuildignore | 1 + r/tools/trim/boost.sh | 205 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 r/tools/trim/boost.sh diff --git a/r/.Rbuildignore b/r/.Rbuildignore index 698ccf3f7afc..faae1aaa4480 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -33,3 +33,4 @@ STYLE.md ^bootstrap.R$ air.toml ^tools/vendored-downloads$ +^tools/trim$ diff --git a/r/tools/trim/boost.sh b/r/tools/trim/boost.sh new file mode 100644 index 000000000000..897da2e76b53 --- /dev/null +++ b/r/tools/trim/boost.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Trim the Boost CMake source tarball (boost-X.Y.Z-cmake.tar.gz, ~140 MB) +# down to the Boost libraries that libarrow's bundled Boost build +# (build_boost() in cpp/cmake_modules/ThirdpartyToolchain.cmake) uses when +# the R package builds libarrow, so it is small enough to ship in the R +# package (~5 MB). +# +# Usage: tools/trim/boost.sh IN_TARBALL OUT_TARBALL +# +# Called by tools/vendor_dependencies.sh (`make sync-cpp`) after tools/cpp +# has been synced. Needs cmake and a C++ compiler, because Boost's own CMake +# is used to work out which libraries are needed. +# +# How it works: +# +# 1. Read the libraries build_boost() adds to BOOST_INCLUDE_LIBRARIES from +# ThirdpartyToolchain.cmake, minus the ones that the R build never needs +# (EXCLUDED_LIBRARIES below). +# 2. Configure Boost with that BOOST_INCLUDE_LIBRARIES. Boost's CMake adds +# the transitive dependencies and logs each library it adds. +# 3. Keep only the top-level files, tools/cmake and those libraries, without +# their tests, docs, examples, etc. +# +# Set BOOST_INCLUDE_LIBRARIES (semicolon-separated) in the environment to +# skip step 1 and use that list instead. + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IN="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +OUT="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" + +# Libraries that build_boost() can add to BOOST_INCLUDE_LIBRARIES but that +# the R package's libarrow build never needs. If you add a library here, +# say why. +# +# - multiprecision: only needed when ARROW_BOOST_NEED_MULTIPRECISION is +# true, i.e. with ARROW_TESTING, ARROW_GANDIVA, ARROW_USE_UBSAN or a +# compiler without a native 128-bit integer (ARROW_USE_NATIVE_INT128). +# The R package builds none of those, and gcc/clang on the 64-bit +# platforms that nixlibs.R builds for all have __int128 (32-bit targets +# don't). Including it would more than double the size of the trimmed +# tarball (~5 MB, 54 libraries -> ~11 MB, 58 libraries). If a build that +# needs it uses this tarball, configuring libarrow fails in build_boost() +# because the boost_multiprecision target doesn't exist. +# - beast, variant, xpressive: only for ARROW_FLIGHT_SQL_ODBC. +EXCLUDED_LIBRARIES=(multiprecision beast variant xpressive) + +# Print the library names that build_boost() adds to BOOST_INCLUDE_LIBRARIES +# with `set(BOOST_INCLUDE_LIBRARIES ...)` or +# `list(APPEND BOOST_INCLUDE_LIBRARIES ...)`, one per line. Variable +# references such as ${ARROW_BOOST_COMPONENTS} are skipped: those are only +# non-empty when building tests, integration tests, benchmarks or ODBC. +boost_libraries_from_cmake() { + awk ' + /^function\(build_boost\)/ { inside = 1; next } + inside && /^endfunction\(\)/ { exit } + inside { + sub(/#.*/, "") + gsub(/\(/, " ( ") + gsub(/\)/, " ) ") + for (i = 1; i <= NF; i++) tok[++n] = $i + } + END { + for (i = 1; i < n; i++) { + if (tok[i + 1] != "(") continue + cmd = tolower(tok[i]) + j = i + 2 + if (cmd == "set" && tok[j] == "BOOST_INCLUDE_LIBRARIES") { + j += 1 + } else if (cmd == "list" && tok[j] == "APPEND" && + tok[j + 1] == "BOOST_INCLUDE_LIBRARIES") { + j += 2 + } else { + continue + } + for (; j <= n && tok[j] != ")"; j++) { + if (tok[j] !~ /^\$/) print tok[j] + } + } + }' "$1" +} + +if [ -z "${BOOST_INCLUDE_LIBRARIES:-}" ]; then + # tools/cpp is the synced copy of cpp/ (`make sync-cpp`); fall back to the + # repository's cpp/ so this can also be run by hand from a git checkout. + TOOLCHAIN="" + for dir in "${SCRIPT_DIR}/../cpp" "${SCRIPT_DIR}/../../../cpp"; do + if [ -f "${dir}/cmake_modules/ThirdpartyToolchain.cmake" ]; then + TOOLCHAIN="${dir}/cmake_modules/ThirdpartyToolchain.cmake" + break + fi + done + if [ -z "${TOOLCHAIN}" ]; then + echo "Can't find ThirdpartyToolchain.cmake; run \`make sync-cpp\` first" >&2 + exit 1 + fi + + ALL_LIBRARIES="$(boost_libraries_from_cmake "${TOOLCHAIN}")" + if [ -z "${ALL_LIBRARIES}" ]; then + echo "No BOOST_INCLUDE_LIBRARIES found in build_boost() in ${TOOLCHAIN}." >&2 + echo "Has build_boost() changed? Update ${BASH_SOURCE[0]}." >&2 + exit 1 + fi + for lib in "${EXCLUDED_LIBRARIES[@]}"; do + if ! printf '%s\n' "${ALL_LIBRARIES}" | grep -qxF "${lib}"; then + echo "Warning: '${lib}' is in EXCLUDED_LIBRARIES but build_boost() no" \ + "longer uses it; remove it from ${BASH_SOURCE[0]}" >&2 + fi + done + BOOST_INCLUDE_LIBRARIES="$(printf '%s\n' "${ALL_LIBRARIES}" | + grep -vxF "$(printf '%s\n' "${EXCLUDED_LIBRARIES[@]}")" | + paste -s -d ';' -)" +fi +echo "Trimming Boost to BOOST_INCLUDE_LIBRARIES=${BOOST_INCLUDE_LIBRARIES} and their dependencies" + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/arrow-trim-boost.XXXXXX")" +trap 'rm -rf "${WORK}"' EXIT + +mkdir "${WORK}/src" "${WORK}/trimmed" +tar -xzf "${IN}" -C "${WORK}/src" +# The tarball has a single top-level directory, e.g. boost-1.92.0 +SRC_NAME="$(ls "${WORK}/src")" +SRC="${WORK}/src/${SRC_NAME}" +if [ ! -f "${SRC}/tools/cmake/include/BoostRoot.cmake" ]; then + echo "${IN} doesn't look like a Boost CMake source tarball" >&2 + exit 1 +fi + +# Let Boost's CMake resolve the transitive dependencies. With +# --log-level=VERBOSE, BoostRoot.cmake logs "Adding Boost library 'x'" for +# each requested library and "Adding Boost dependency 'x'" for each +# dependency (including 'headers', which is always added). +# BUILD_TESTING=OFF matches prepare_fetchcontent(), so test-only +# dependencies are not pulled in. +cmake -S "${SRC}" -B "${WORK}/build" \ + --log-level=VERBOSE \ + -DBUILD_TESTING=OFF \ + -DBOOST_INCLUDE_LIBRARIES="${BOOST_INCLUDE_LIBRARIES}" \ + > "${WORK}/cmake.log" 2>&1 || { + cat "${WORK}/cmake.log" >&2 + echo "Configuring Boost failed" >&2 + exit 1 +} +if grep -q "has not been found" "${WORK}/cmake.log"; then + grep "has not been found" "${WORK}/cmake.log" >&2 + echo "A library in BOOST_INCLUDE_LIBRARIES doesn't exist in ${SRC_NAME}" >&2 + exit 1 +fi +# (bash 3.2 on macOS has no mapfile) +KEEP_LIBRARIES=() +while IFS= read -r lib; do + KEEP_LIBRARIES+=("${lib}") +done < <(sed -n "s/.*Adding Boost [a-z]* '\([^']*\)'.*/\1/p" "${WORK}/cmake.log" | LC_ALL=C sort -u) +if [ "${#KEEP_LIBRARIES[@]}" -eq 0 ]; then + echo "Found no \"Adding Boost ...\" messages in Boost's CMake output." >&2 + echo "Has BoostRoot.cmake changed? Update ${BASH_SOURCE[0]}." >&2 + exit 1 +fi +echo "Keeping ${#KEEP_LIBRARIES[@]} Boost libraries: ${KEEP_LIBRARIES[*]}" + +# Copy what the bundled build needs into ${WORK}/trimmed/${SRC_NAME}: +# the top-level files (CMakeLists.txt, LICENSE_1_0.txt, ...), tools/cmake +# (BoostRoot.cmake etc.) and the libraries, but not the top-level doc/, +# more/, status/ or the other tools/. +DEST="${WORK}/trimmed/${SRC_NAME}" +mkdir -p "${DEST}/tools" +find "${SRC}" -maxdepth 1 -type f -exec cp -p {} "${DEST}/" \; +cp -pR "${SRC}/tools/cmake" "${DEST}/tools/" +for lib in "${KEEP_LIBRARIES[@]}"; do + # lib can be a nested library such as numeric/conversion + mkdir -p "$(dirname "${DEST}/libs/${lib}")" + cp -pR "${SRC}/libs/${lib}" "${DEST}/libs/${lib}" + # Only remove these at the top level of each library: deeper down they + # can be header directories (e.g. include/boost/math/tools/). + for dir in test doc example examples bench benchmark perf performance meta tools; do + rm -rf "${DEST}/libs/${lib:?}/${dir}" + done +done + +# COPYFILE_DISABLE stops macOS tar from adding AppleDouble (._*) files. +(cd "${WORK}/trimmed" && COPYFILE_DISABLE=1 tar -czf "${OUT}" "${SRC_NAME}")