From bdd7fbf714809c092cbd20a0627756e18466c501 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 18:41:39 -0600 Subject: [PATCH 1/2] GH-55478: [R] Vendor thirdparty sources needed for Parquet, codecs, compute and JSON Ship trimmed source tarballs of the thirdparty dependencies needed for Parquet, the compression codecs, RE2/utf8proc, JSON and mimalloc in the R package so that offline source builds (e.g. on CRAN) can include those features instead of falling back to a minimal build. * tools/vendor_dependencies.sh (run by `make sync-cpp`) downloads, trims and vendors the tarballs into tools/vendored, and records each one's upstream and trimmed checksums in tools/vendored/checksums.txt. A dependency can provide its own trim script at tools/trim/.sh. * When building the package's copy of the C++ source (tools/cpp), nixlibs.R uses the vendored tarballs that were trimmed from the versions that versions.txt pins, replacing their checksums in versions.txt with the trimmed ones. When offline, it only turns off the features whose dependencies aren't vendored. * The offline CI job (now test-r-offline-vendored) checks that the expected features are available after the build. * Add inst/COPYRIGHTS listing the vendored dependencies' licenses. Co-Authored-By: Claude Opus 5.5 --- ci/scripts/r_test.sh | 20 +++ dev/release/rat_exclude_files.txt | 1 + dev/tasks/tasks.yml | 30 +++- r/.Rbuildignore | 1 + r/.gitignore | 2 + r/DESCRIPTION | 1 + r/inst/COPYRIGHTS | 78 ++++++++++ r/tools/nixlibs.R | 217 ++++++++++++++++++++++---- r/tools/test-nixlibs.R | 227 ++++++++++++++++++++++----- r/tools/vendor_dependencies.sh | 248 +++++++++++++++++++++++++++--- 10 files changed, 727 insertions(+), 98 deletions(-) create mode 100644 r/inst/COPYRIGHTS diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 20e6062e847f..ac9fbf2c479d 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -158,6 +158,26 @@ if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then cp /etc/resolv.conf{.bak,} fi +if [ -n "${ARROW_R_EXPECTED_FEATURES:-}" ]; then + # The tests skip what the build doesn't support, so make sure the features + # we expect (e.g. from the vendored thirdparty sources) were actually built. + # ARROW_R_EXPECTED_FEATURES is a comma-separated list of + # names(arrow::arrow_info()$capabilities) and memory pool backends + # (e.g. mimalloc). + ${R_BIN} -s -e " + library(arrow, lib.loc = 'check/arrow.Rcheck') + info <- arrow_info() + print(info\$capabilities) + print(info\$memory_pool\$available_backends) + available <- c(names(which(info\$capabilities)), info\$memory_pool\$available_backends) + expected <- strsplit(Sys.getenv('ARROW_R_EXPECTED_FEATURES'), ',')[[1]] + missing <- setdiff(expected, available) + if (length(missing) > 0) { + stop('Expected features are not available: ', paste(missing, collapse = ', ')) + } + " +fi + AFTER=$(ls -alh ~/) if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then # Ignore ~/.TinyTex/ and ~/R/ because it has many files. diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 3c5518a9596e..ef6e1a4f88dd 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -97,6 +97,7 @@ r/tests/testthat/_snaps/*.md r/inst/include/cpp11.hpp r/inst/include/cpp11/*.hpp r/tools/nixlibs-allowlist.txt +r/inst/COPYRIGHTS .gitattributes ruby/red-arrow/.yardopts ruby/red-arrow-format/lib/arrow-format/org/* diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d9351d3afe0a..8ea23b6d93b4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -633,14 +633,30 @@ tasks: template: r/github.linux.offline.build.yml # Simulate a CRAN builder without network access: the build should detect - # that it is offline and fall back to a minimal build without downloading - # anything. - test-r-offline-minimal: - ci: github - template: docker-tests/github.linux.yml - params: - flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true' + # that it is offline and build libarrow without downloading anything, using + # the thirdparty sources vendored in the package (tools/vendored) to still + # include Parquet, Dataset, JSON, the compression codecs, etc. + # ARROW_SOURCE_HOME='' makes it use the C++ source copied into tools/cpp + # (like the CRAN package does), whose versions.txt has the checksums of the + # trimmed tarballs. The codecs that are off by default on Linux are turned on + # so that every vendored tarball is built. + test-r-offline-vendored: + ci: github + template: docker-tests/github.linux.yml + params: + flags: >- + -e ARROW_R_TEST_OFFLINE=true + -e NOT_CRAN=false + -e SKIP_VIGNETTES=true + -e ARROW_SOURCE_HOME='' + -e ARROW_WITH_BROTLI=ON + -e ARROW_WITH_BZ2=ON + -e ARROW_WITH_ZSTD=ON + -e ARROW_R_EXPECTED_FEATURES=dataset,parquet,json,mimalloc,re2,utf8proc,snappy,gzip,brotli,zstd,lz4,bz2 image: r + # A full libarrow build with 2 cores (NOT_CRAN=false) and no sccache + # (it's offline) + timeout: 180 # This one has -flto=auto test-r-ubuntu-22.04: diff --git a/r/.Rbuildignore b/r/.Rbuildignore index deaf5dd3948c..698ccf3f7afc 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -32,3 +32,4 @@ STYLE.md ^inst/__pycache__$ ^bootstrap.R$ air.toml +^tools/vendored-downloads$ diff --git a/r/.gitignore b/r/.gitignore index a1295ea4e0b5..e16bc75f459b 100644 --- a/r/.gitignore +++ b/r/.gitignore @@ -35,3 +35,5 @@ extra-tests/files # Source tarballs of required dependencies that we ship for offline builds. # They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh). /tools/vendored/ +# Cache of the upstream (untrimmed) tarballs used by tools/vendor_dependencies.sh +/tools/vendored-downloads/ diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 1e5733c37cd5..19918beab685 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -24,6 +24,7 @@ Description: 'Apache' 'Arrow' is a cross-language package provides an interface to the 'Arrow C++' library. Depends: R (>= 4.2) License: Apache License (>= 2.0) +Copyright: See file inst/COPYRIGHTS URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/ BugReports: https://github.com/apache/arrow/issues Encoding: UTF-8 diff --git a/r/inst/COPYRIGHTS b/r/inst/COPYRIGHTS new file mode 100644 index 000000000000..779b8b2a5fa8 --- /dev/null +++ b/r/inst/COPYRIGHTS @@ -0,0 +1,78 @@ +The arrow R package is Copyright The Apache Software Foundation and is licensed +under the Apache License, Version 2.0. The Arrow C++ library source copied into +tools/cpp includes code from other projects; see tools/LICENSE.txt and +tools/NOTICE.txt in the source package. + +To build the Arrow C++ library without network access, the source package also +includes (trimmed) source tarballs of these thirdparty dependencies in +tools/vendored. They are only used when the dependency isn't found on the +system, and are statically linked into the package when used. Each tarball +contains the dependency's full license text. + +Boost (headers used by Apache Thrift) + Copyright the Boost authors + Boost Software License 1.0 + +Brotli + Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors + MIT License + +bzip2 + Copyright (C) 1996-2019 Julian R Seward + BSD-style license (see LICENSE in the tarball) + +LZ4 (lib/ only) + Copyright (c) 2011-2020, Yann Collet + BSD 2-Clause License + +mimalloc + Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen + MIT License + +RapidJSON + Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip + MIT License (with code under the BSD and JSON licenses, see license.txt in + the tarball) + +RE2 + Copyright (c) 2009 The RE2 Authors + BSD 3-Clause License + +simdjson + Copyright 2018-2025 The simdjson authors + Apache License 2.0 or MIT License + +Snappy + Copyright 2011, Google Inc. + BSD 3-Clause License + +Apache Thrift + Copyright (C) 2006-2019, The Apache Software Foundation + Apache License 2.0 + +uriparser (library sources only) + Copyright (C) 2007, Weijia Song and Sebastian Pipping + BSD 3-Clause License + +utf8proc + Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Tony Kelman, + Jonas Fonseca, and other contributors + Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany + MIT License, with Unicode data under the Unicode Data Files and Software + License + +xsimd + Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and + Martin Renou + Copyright (c) 2016, QuantStack + Copyright (c) 2018, Serge Guelton + BSD 3-Clause License + +zlib + Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler + zlib License + +Zstandard (zstd) + Copyright (c) Meta Platforms, Inc. and affiliates + BSD 3-Clause License (zstd is dual-licensed BSD/GPLv2; it is used under the + BSD license) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 226894372ec5..a2c321647bd5 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -486,7 +486,7 @@ find_local_source <- function() { # The second probably occurs if we're installing the arrow R package cpp_dir_options <- c( file.path(Sys.getenv("ARROW_SOURCE_HOME", ".."), "cpp"), - "tools/cpp" + package_cpp_dir ) for (cpp_dir in cpp_dir_options) { if (file.exists(file.path(cpp_dir, "src/arrow/api.h"))) { @@ -625,30 +625,35 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- with_cloud_support(env_var_list) env_var_list <- with_wasm_support(env_var_list) - # turn_off_all_optional_features() needs to happen after - # with_cloud_support(), since it might turn features ON. + # turn_off_all_optional_features() and turn_off_unvendored_features() need to + # happen after with_cloud_support(), since it might turn features ON. thirdparty_deps_unavailable <- !download_ok && !dir.exists(thirdparty_dependency_dir) && !env_is("ARROW_DEPENDENCY_SOURCE", "system") do_minimal_build <- env_is("LIBARROW_MINIMAL", "true") + # Thirdparty source tarballs shipped in the package + vendored <- usable_vendored_dependencies(src_dir) if (do_minimal_build) { env_var_list <- turn_off_all_optional_features(env_var_list) } else if (thirdparty_deps_unavailable) { cat(paste0( "*** Building C++ library from source, but downloading thirdparty dependencies\n", - " is not possible, so this build will turn off all thirdparty features.\n", + " is not possible, so this build will turn off the thirdparty features\n", + " whose sources are not included in the package.\n", " See installation guide for details:\n", " https://arrow.apache.org/docs/r/articles/install.html\n" )) - env_var_list <- turn_off_all_optional_features(env_var_list) + env_var_list <- turn_off_unvendored_features(env_var_list, names(vendored)) } else if (dir.exists(thirdparty_dependency_dir)) { # Add the *_SOURCE_URL env vars env_var_list <- set_thirdparty_urls(env_var_list) } - # Some thirdparty dependencies are required even for a minimal build, so they - # can't be turned off like the ones above. - env_var_list <- with_vendored_dependencies(env_var_list) + # Point the build at the vendored sources of the thirdparty dependencies. + # Some of them are required even for a minimal build, so they can't be turned + # off like the ones above. + env_var_list <- with_vendored_dependencies(env_var_list, vendored) + use_vendored_checksums(src_dir, env_var_list, vendored) env_vars <- env_vars_as_string(env_var_list) lg("arrow %s", ifelse(quietly, "", paste("with", env_vars)), .indent = "****") @@ -795,7 +800,9 @@ cmake_version <- function(cmd = "cmake") { ) } -turn_off_all_optional_features <- function(env_var_list) { +# `keep` is a vector of features (names in `turn_off` below, or +# "ARROW_SIMD_LEVEL") to leave as they are. +turn_off_all_optional_features <- function(env_var_list, keep = character()) { # Because these are done as environment variables (as opposed to build flags), # setting these to "OFF" overrides any previous setting. We don't need to # check the existing value. @@ -822,19 +829,53 @@ turn_off_all_optional_features <- function(env_var_list) { "ARROW_WITH_ZLIB" = "OFF", "ARROW_WITH_ZSTD" = "OFF", "ARROW_WITH_RE2" = "OFF", - "ARROW_WITH_UTF8PROC" = "OFF", + "ARROW_WITH_UTF8PROC" = "OFF" + ) + turn_off <- turn_off[setdiff(names(turn_off), keep)] + if (!("ARROW_SIMD_LEVEL" %in% keep)) { # The syntax to turn off XSIMD is different. # Pull existing value of EXTRA_CMAKE_FLAGS first (must be defined) - "EXTRA_CMAKE_FLAGS" = paste( - env_var_list[["EXTRA_CMAKE_FLAGS"]], - "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE" + turn_off <- c( + turn_off, + "EXTRA_CMAKE_FLAGS" = paste( + env_var_list[["EXTRA_CMAKE_FLAGS"]], + "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE" + ) ) - ) + } # Create a new env_var_list, with the values of turn_off set. # replace() also adds new values if they didn't exist before replace(env_var_list, names(turn_off), turn_off) } +# Without network access, libarrow can only build the thirdparty dependencies +# whose source tarballs are shipped in the package (see +# `with_vendored_dependencies()`), so turn off every optional feature except +# those whose dependencies are all vendored. Features that are kept are left as +# they are (e.g. as the user or the platform defaults set them), not turned on. +# `available` is the ARROW_*_URL names of the usable vendored tarballs (see +# `usable_vendored_dependencies()`). +turn_off_unvendored_features <- function(env_var_list, available) { + has_all <- vapply(vendored_features, function(deps) all(deps %in% available), logical(1)) + keep <- names(vendored_features)[has_all] + if ("ARROW_PARQUET" %in% keep) { + # Dataset needs no thirdparty dependencies itself, but it's only useful + # with Parquet (and we've always turned them on and off together) + keep <- c(keep, "ARROW_DATASET") + } + if ("ARROW_XSIMD_URL" %in% available) { + keep <- c(keep, "ARROW_SIMD_LEVEL") + } + if (!quietly) { + lg( + "Features with vendored dependencies: %s", + if (length(keep)) paste(keep, collapse = ", ") else "none", + .indent = "****" + ) + } + turn_off_all_optional_features(env_var_list, keep = keep) +} + get_component_names <- function() { if (!isTRUE(Sys.which("bash") != "")) { stop("nixlibs.R requires bash to be installed and available in your PATH") @@ -874,26 +915,53 @@ set_thirdparty_urls <- function(env_var_list) { env_var_list } -# libarrow requires some thirdparty dependencies, even for a minimal build. -# The libarrow build uses them from the system if they are available and -# otherwise builds them from source, downloading from ARROW_*_URL. To make sure -# that works without network access, `make sync-cpp` saves their source -# tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here. +# The libarrow build uses thirdparty dependencies from the system if they are +# available and otherwise builds them from source, downloading from ARROW_*_URL. +# Some are required even for a minimal build. To make sure that works without +# network access, `make sync-cpp` saves (trimmed) source tarballs into +# vendored_dependency_dir and we point ARROW_*_URL at them here. # This does not change whether the system versions are used. -with_vendored_dependencies <- function(env_var_list) { - if (!dir.exists(vendored_dependency_dir)) { +# `vendored` is the result of `usable_vendored_dependencies()`. +with_vendored_dependencies <- function(env_var_list, vendored) { + for (env_varname in names(vendored)) { + if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { + # Respect the user's (or set_thirdparty_urls()'s) choice + next + } + env_var_list <- replace(env_var_list, env_varname, vendored[[env_varname]]) + } + env_var_list +} + +# Returns the paths of the vendored tarballs that the libarrow source in +# `src_dir` can use, named by their ARROW_*_URL. +# The tarballs are trimmed, so the libarrow build only accepts them once +# `use_vendored_checksums()` has put their checksums in versions.txt. We only +# do that to the package's own copy of the source (tools/cpp), not e.g. ../cpp +# of a git checkout, and only for tarballs that were trimmed from the version +# that versions.txt pins (according to the checksums.txt file that +# tools/vendor_dependencies.sh writes). +usable_vendored_dependencies <- function(src_dir) { + manifest <- read_vendored_checksums() + if (is.null(manifest)) { # e.g. building from a git checkout without `make sync-cpp` if (!quietly) { lg("No vendored dependencies found, they will be built with CMake if needed", .indent = "****") } - return(env_var_list) + return(character()) } + is_package_cpp_dir <- !is.null(src_dir) && + identical(normalizePath(src_dir, mustWork = FALSE), normalizePath(package_cpp_dir, mustWork = FALSE)) + if (!is_package_cpp_dir) { + if (!quietly) { + lg("Not using vendored dependencies, they are only used to build %s", package_cpp_dir, .indent = "****") + } + return(character()) + } + expected <- read_thirdparty_checksums(file.path(src_dir, "thirdparty", "versions.txt")) deps_df <- get_component_names() + usable <- character() for (env_varname in vendored_dependencies) { - if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { - # Respect the user's (or set_thirdparty_urls()'s) choice - next - } filename <- deps_df$filename[deps_df$env_varname == env_varname] tarball <- file.path(vendored_dependency_dir, filename) if (length(tarball) != 1 || !file.exists(tarball)) { @@ -902,9 +970,68 @@ with_vendored_dependencies <- function(env_var_list) { } next } - env_var_list <- replace(env_var_list, env_varname, normalizePath(tarball)) + checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) + entry <- manifest[manifest$filename == filename, , drop = FALSE] + # versions.txt has the vendored checksum if use_vendored_checksums() has + # already run on it (e.g. when installing from the r/ directory again) + matches <- nrow(entry) == 1 && + unname(expected[checksum_varname]) %in% c(entry$upstream_sha256, entry$vendored_sha256) + if (!matches) { + if (!quietly) { + lg( + "Vendored source for %s doesn't match %s, it will be downloaded if needed", + env_varname, + file.path(src_dir, "thirdparty", "versions.txt"), + .indent = "****" + ) + } + next + } + usable[[env_varname]] <- normalizePath(tarball) } - env_var_list + usable +} + +# Read the checksums.txt that tools/vendor_dependencies.sh writes, with the +# checksums of each vendored tarball before (upstream) and after trimming +read_vendored_checksums <- function() { + manifest_file <- file.path(vendored_dependency_dir, "checksums.txt") + if (!file.exists(manifest_file)) { + return(NULL) + } + read.table(manifest_file, header = TRUE, stringsAsFactors = FALSE) +} + +# The libarrow build checks the checksums of the thirdparty tarballs against +# src_dir's versions.txt, so replace the upstream checksums of the vendored +# tarballs that `env_var_list` points at with the trimmed tarballs' checksums. +use_vendored_checksums <- function(src_dir, env_var_list, vendored) { + used <- vendored[vapply( + names(vendored), + function(env_varname) identical(env_var_list[[env_varname]], vendored[[env_varname]]), + logical(1) + )] + if (length(used) == 0) { + return(invisible()) + } + manifest <- read_vendored_checksums() + versions_txt <- file.path(src_dir, "thirdparty", "versions.txt") + lines <- readLines(versions_txt) + for (env_varname in names(used)) { + checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) + checksum <- manifest$vendored_sha256[manifest$filename == basename(used[[env_varname]])] + lines <- sub(paste0("^", checksum_varname, "=.*$"), paste0(checksum_varname, "=", checksum), lines) + } + writeLines(lines, versions_txt) +} + +# Parse the *_BUILD_SHA256_CHECKSUM=... lines of a versions.txt into a named vector +read_thirdparty_checksums <- function(versions_txt) { + if (!file.exists(versions_txt)) { + return(character()) + } + lines <- grep("^[A-Z0-9_]+_BUILD_SHA256_CHECKSUM=", readLines(versions_txt), value = TRUE) + setNames(sub("^[^=]*=", "", lines), sub("=.*$", "", lines)) } # this is generally about features that people asked for via environment variables, but @@ -1140,11 +1267,37 @@ if (download_libarrow_ok && !is_release && !test_mode) { # `create_package_with_all_dependencies()` in install-arrow.R thirdparty_dependency_dir <- Sys.getenv("ARROW_THIRDPARTY_DEPENDENCY_DIR", "tools/thirdparty_dependencies") -# Source tarballs of dependencies that libarrow requires even for a minimal build. -# These are always shipped in the package (see `with_vendored_dependencies()`). -# If you change this path or list, you also need to edit tools/vendor_dependencies.sh +# Trimmed source tarballs of thirdparty dependencies, shipped in the package so +# that building without network access still works and can include the features +# below (see `with_vendored_dependencies()` and `turn_off_unvendored_features()`). +# If you change this path or these lists, you also need to edit +# tools/vendor_dependencies.sh vendored_dependency_dir <- "tools/vendored" -vendored_dependencies <- c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL") +# The copy of the libarrow source in the package (see `make sync-cpp`) +package_cpp_dir <- "tools/cpp" +# Optional features and the ARROW_*_URL of each dependency they need (when the +# dependency is built from source). See "Some EP's require other EP's" in +# cpp/cmake_modules/ThirdpartyToolchain.cmake +vendored_features <- list( + # Parquet turns on Thrift and simdjson; Thrift needs zlib and Boost + ARROW_PARQUET = c("ARROW_BOOST_URL", "ARROW_SIMDJSON_URL", "ARROW_THRIFT_URL", "ARROW_ZLIB_URL"), + ARROW_JSON = c("ARROW_RAPIDJSON_URL", "ARROW_SIMDJSON_URL"), + ARROW_MIMALLOC = "ARROW_MIMALLOC_URL", + ARROW_WITH_BROTLI = "ARROW_BROTLI_URL", + ARROW_WITH_BZ2 = "ARROW_BZIP2_URL", + ARROW_WITH_LZ4 = "ARROW_LZ4_URL", + ARROW_WITH_RE2 = "ARROW_RE2_URL", + ARROW_WITH_SNAPPY = "ARROW_SNAPPY_URL", + ARROW_WITH_UTF8PROC = "ARROW_UTF8PROC_URL", + ARROW_WITH_ZLIB = "ARROW_ZLIB_URL", + ARROW_WITH_ZSTD = "ARROW_ZSTD_URL" +) +# uriparser and xsimd are required even for a minimal build +vendored_dependencies <- unique(c( + "ARROW_URIPARSER_URL", + "ARROW_XSIMD_URL", + unlist(vendored_features, use.names = FALSE) +)) arrow_versioned <- paste0("arrow-", VERSION) # configure.win uses a different libarrow dir and the zip is already nested diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index 9d8abccfbb5e..0320658d41d4 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -233,58 +233,213 @@ test_that("find_latest_nightly()", { ) }) -test_that("with_vendored_dependencies()", { - vendored_dir <- tempfile() - dir.create(vendored_dir) - on.exit(unlink(vendored_dir, recursive = TRUE), add = TRUE) - original_dir <- nixlibs_env$vendored_dependency_dir - original_get_component_names <- nixlibs_env$get_component_names - original_quietly <- nixlibs_env$quietly - on.exit(nixlibs_env$vendored_dependency_dir <- original_dir, add = TRUE) - on.exit(nixlibs_env$get_component_names <- original_get_component_names, add = TRUE) - on.exit(nixlibs_env$quietly <- original_quietly, add = TRUE) - nixlibs_env$vendored_dependency_dir <- vendored_dir - nixlibs_env$get_component_names <- function() { - data.frame( - env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL"), - filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz") +# Set up a fake vendored_dependency_dir and package_cpp_dir for the vendored +# dependency tests. `vendored` is a named vector of filename = upstream +# checksum in checksums.txt (the trimmed checksum is "-trimmed"), +# `expected` is the checksums in the source dir's versions.txt, named by +# ARROW_*_URL. +local_vendored_dependencies <- function(vendored, expected, env = parent.frame()) { + vendored_dir <- withr::local_tempdir(.local_envir = env) + src_dir <- withr::local_tempdir(.local_envir = env) + for (filename in names(vendored)) { + file.create(file.path(vendored_dir, filename)) + } + if (length(vendored)) { + writeLines( + c( + "filename upstream_sha256 vendored_sha256", + paste(names(vendored), vendored, paste0(vendored, "-trimmed")) + ), + file.path(vendored_dir, "checksums.txt") ) } - withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) + dir.create(file.path(src_dir, "thirdparty")) + writeLines( + c( + "# a comment", + paste0(sub("_URL$", "_BUILD_SHA256_CHECKSUM", names(expected)), "=", expected) + ), + file.path(src_dir, "thirdparty", "versions.txt") + ) + local_binding("vendored_dependency_dir", vendored_dir, env) + local_binding("package_cpp_dir", src_dir, env) + local_binding( + "get_component_names", + function() { + data.frame( + env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL", "ARROW_JEMALLOC_URL"), + filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz", "jemalloc-5.3.0.tar.bz2") + ) + }, + env + ) + list(vendored_dir = vendored_dir, src_dir = src_dir) +} + +# Temporarily replace a variable in nixlibs_env +local_binding <- function(name, value, env = parent.frame()) { + original <- nixlibs_env[[name]] + assign(name, value, envir = nixlibs_env) + withr::defer(assign(name, original, envir = nixlibs_env), envir = env) +} - # Without the vendored tarballs, nothing is set - nixlibs_env$quietly <- TRUE - expect_silent( - expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")) +test_that("usable_vendored_dependencies()", { + local_binding("quietly", TRUE) + # Nothing vendored + dirs <- local_vendored_dependencies(character(), c(ARROW_URIPARSER_URL = "aaa")) + expect_identical(usable_vendored_dependencies(dirs$src_dir), character()) + + dirs <- local_vendored_dependencies( + c( + "uriparser-1.0.2.tar.bz2" = "aaa", + "xsimd-14.3.0.tar.gz" = "bbb", + "zstd-1.5.7.tar.gz" = "ccc", + # Not in vendored_dependencies, so it isn't used + "jemalloc-5.3.0.tar.bz2" = "ddd" + ), + c( + ARROW_URIPARSER_URL = "aaa", + # A different version than the vendored one + ARROW_XSIMD_URL = "other", + # Already has the trimmed checksum (use_vendored_checksums() has run) + ARROW_ZSTD_URL = "ccc-trimmed", + ARROW_JEMALLOC_URL = "ddd" + ) + ) + expect_identical( + usable_vendored_dependencies(dirs$src_dir), + c( + ARROW_URIPARSER_URL = normalizePath(file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2")), + ARROW_ZSTD_URL = normalizePath(file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz")) + ) + ) + # Mismatches are only reported with verbose logging + local_binding("quietly", FALSE) + expect_output( + usable_vendored_dependencies(dirs$src_dir), + "Vendored source for ARROW_XSIMD_URL doesn't match" + ) + + # Only used to build package_cpp_dir (not e.g. ../cpp of a git checkout) + other_src_dir <- withr::local_tempdir() + file.copy(file.path(dirs$src_dir, "thirdparty"), other_src_dir, recursive = TRUE) + expect_output( + expect_identical(usable_vendored_dependencies(other_src_dir), character()), + "Not using vendored dependencies" + ) + expect_output( + expect_identical(usable_vendored_dependencies(NULL), character()), + "Not using vendored dependencies" ) - # The missing tarballs are only reported with verbose logging - nixlibs_env$quietly <- FALSE + + # A missing tarball isn't used, even if it's in checksums.txt + unlink(file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz")) expect_output( - expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")), - "Vendored source for ARROW_URIPARSER_URL not found" + expect_named(usable_vendored_dependencies(dirs$src_dir), "ARROW_URIPARSER_URL"), + "Vendored source for ARROW_ZSTD_URL not found" ) - nixlibs_env$quietly <- TRUE +}) - uriparser <- file.path(vendored_dir, "uriparser-1.0.2.tar.bz2") - xsimd <- file.path(vendored_dir, "xsimd-14.3.0.tar.gz") - # Only dependencies in vendored_dependencies are used - zstd <- file.path(vendored_dir, "zstd-1.5.7.tar.gz") - file.create(uriparser, xsimd, zstd) +test_that("use_vendored_checksums()", { + local_binding("quietly", TRUE) + dirs <- local_vendored_dependencies( + c("uriparser-1.0.2.tar.bz2" = "aaa", "xsimd-14.3.0.tar.gz" = "bbb", "zstd-1.5.7.tar.gz" = "ccc"), + c(ARROW_URIPARSER_URL = "aaa", ARROW_XSIMD_URL = "bbb", ARROW_ZSTD_URL = "ccc", ARROW_JEMALLOC_URL = "ddd") + ) + versions_txt <- file.path(dirs$src_dir, "thirdparty", "versions.txt") + original <- readLines(versions_txt) + vendored <- c( + ARROW_URIPARSER_URL = file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2"), + ARROW_XSIMD_URL = file.path(dirs$vendored_dir, "xsimd-14.3.0.tar.gz"), + ARROW_ZSTD_URL = file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz") + ) + + # Nothing is pointed at the vendored tarballs + use_vendored_checksums(dirs$src_dir, list(), vendored) + expect_identical(readLines(versions_txt), original) + + # Only the ones that are (e.g. not when the user set ARROW_XSIMD_URL) + env_var_list <- list( + ARROW_URIPARSER_URL = vendored[["ARROW_URIPARSER_URL"]], + ARROW_XSIMD_URL = "https://example.com/xsimd.tar.gz", + ARROW_ZSTD_URL = vendored[["ARROW_ZSTD_URL"]] + ) + use_vendored_checksums(dirs$src_dir, env_var_list, vendored) expect_identical( - with_vendored_dependencies(list(A = "1")), - list(A = "1", ARROW_URIPARSER_URL = normalizePath(uriparser), ARROW_XSIMD_URL = normalizePath(xsimd)) + readLines(versions_txt), + c( + "# a comment", + "ARROW_URIPARSER_BUILD_SHA256_CHECKSUM=aaa-trimmed", + "ARROW_XSIMD_BUILD_SHA256_CHECKSUM=bbb", + "ARROW_ZSTD_BUILD_SHA256_CHECKSUM=ccc-trimmed", + "ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=ddd" + ) + ) + # and they are still usable afterwards + expect_named( + usable_vendored_dependencies(dirs$src_dir), + c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL") + ) +}) + +test_that("with_vendored_dependencies()", { + vendored <- c(ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") + withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) + + expect_identical(with_vendored_dependencies(list(A = "1"), character()), list(A = "1")) + expect_identical( + with_vendored_dependencies(list(A = "1"), vendored), + list(A = "1", ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") ) # Already set (e.g. by set_thirdparty_urls()) is respected expect_identical( - with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2")), - list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = normalizePath(xsimd)) + with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2"), vendored), + list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") ) # As is a user-set env var withr::local_envvar(c(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2")) expect_identical( - with_vendored_dependencies(list(A = "1")), - list(A = "1", ARROW_XSIMD_URL = normalizePath(xsimd)) + with_vendored_dependencies(list(A = "1"), vendored), + list(A = "1", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") + ) +}) + +test_that("turn_off_unvendored_features()", { + local_binding("quietly", TRUE) + env_var_list <- list(EXTRA_CMAKE_FLAGS = "", ARROW_WITH_ZSTD = "ON", ARROW_S3 = "ON") + + # Nothing vendored: the same as turn_off_all_optional_features() + expect_identical( + turn_off_unvendored_features(env_var_list, character()), + turn_off_all_optional_features(env_var_list) + ) + + # Everything vendored: only features that need unvendored dependencies are + # turned off, and the rest are left as they were (not turned on) + result <- turn_off_unvendored_features(env_var_list, vendored_dependencies) + expect_identical(result$ARROW_S3, "OFF") + expect_identical(result$ARROW_JEMALLOC, "OFF") + expect_identical(result$ARROW_WITH_ZSTD, "ON") + for (feature in setdiff(c(names(vendored_features), "ARROW_DATASET"), "ARROW_WITH_ZSTD")) { + expect_null(result[[feature]]) + } + expect_identical(result$EXTRA_CMAKE_FLAGS, "") + + # Parquet (and Dataset) need all of their dependencies + result <- turn_off_unvendored_features( + env_var_list, + c("ARROW_XSIMD_URL", "ARROW_THRIFT_URL", "ARROW_ZSTD_URL") + ) + expect_identical(result$ARROW_PARQUET, "OFF") + expect_identical(result$ARROW_DATASET, "OFF") + expect_identical(result$ARROW_WITH_ZSTD, "ON") + expect_identical(result$ARROW_WITH_SNAPPY, "OFF") + # SIMD is only turned off if xsimd isn't vendored + expect_identical(result$EXTRA_CMAKE_FLAGS, "") + expect_match( + turn_off_unvendored_features(env_var_list, "ARROW_ZSTD_URL")$EXTRA_CMAKE_FLAGS, + "-DARROW_SIMD_LEVEL=NONE" ) }) diff --git a/r/tools/vendor_dependencies.sh b/r/tools/vendor_dependencies.sh index caa28bff5ce2..686ce981e286 100644 --- a/r/tools/vendor_dependencies.sh +++ b/r/tools/vendor_dependencies.sh @@ -17,30 +17,80 @@ # specific language governing permissions and limitations # under the License. -# libarrow requires some thirdparty dependencies, even for a minimal build. -# This script downloads the source tarballs that libarrow would otherwise -# download at build time, so that they can be shipped in the R package. -# nixlibs.R uses them when the dependencies are not found on the system, so -# that building without network access (e.g. on CRAN) still works. +# libarrow downloads the source of its thirdparty dependencies at build time +# when they aren't found on the system. This script downloads those source +# tarballs ahead of time so that they can be shipped in the R package, which +# lets building without network access (e.g. on CRAN) still include features +# like Parquet. nixlibs.R uses them when the dependencies are not found on the +# system, and turns off the features whose dependencies aren't vendored. +# +# To keep the package small, each tarball is trimmed down to what the libarrow +# build uses (no tests, docs, other language bindings, ...). Trimmed tarballs +# no longer match the checksums pinned in versions.txt, so this script records +# the upstream and trimmed checksums of each tarball in DESTDIR/checksums.txt. +# When building the package's copy of the libarrow source (tools/cpp), nixlibs.R +# uses the tarballs whose upstream checksum matches versions.txt and replaces +# it with the trimmed one. +# +# Trimming is done by, in order of preference: +# 1. tools/trim/.sh IN_TARBALL OUT_TARBALL, if it exists, where +# is the lowercase dependency name (e.g. ARROW_BOOST_URL -> boost). Use +# this for dependencies that need more than a few `rm`s. +# 2. Otherwise, trim_generic() plus the dependency-specific removals in +# trim_dependency() below. # # Usage: tools/vendor_dependencies.sh [DESTDIR] -# Must be run after tools/cpp has been synced (see `make sync-cpp`). +# Must be run after tools/cpp has been synced (see `make sync-cpp`). Needs curl +# and cmake (to handle .tar.bz2 without a bzip2 binary). +# +# Environment variables: +# ARROW_VENDOR_DOWNLOAD_DIR: where to cache the upstream (untrimmed) +# tarballs between runs (default: tools/vendored-downloads, which is +# ignored by git and R CMD build). set -euo pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DESTDIR="${1:-${SOURCE_DIR}/vendored}" +DOWNLOAD_DIR="${ARROW_VENDOR_DOWNLOAD_DIR:-${SOURCE_DIR}/vendored-downloads}" +VERSIONS_TXT="${SOURCE_DIR}/cpp/thirdparty/versions.txt" # If you change this list, you also need to edit -# `vendored_dependencies` in nixlibs.R +# `vendored_dependencies` (and `vendored_features`) in nixlibs.R VENDORED_DEPENDENCIES=( + # Required even for a minimal build ARROW_URIPARSER_URL ARROW_XSIMD_URL + # Parquet (ARROW_PARQUET turns on Thrift and simdjson; Thrift needs zlib and + # Boost headers) + ARROW_BOOST_URL + ARROW_SIMDJSON_URL + ARROW_THRIFT_URL + ARROW_ZLIB_URL + # Compression codecs + ARROW_BROTLI_URL + ARROW_BZIP2_URL + ARROW_LZ4_URL + ARROW_SNAPPY_URL + ARROW_ZSTD_URL + # Compute (RE2, utf8proc), JSON (RapidJSON, simdjson) and the allocator + ARROW_MIMALLOC_URL + ARROW_RAPIDJSON_URL + ARROW_RE2_URL + ARROW_UTF8PROC_URL ) +if [ ! -f "${VERSIONS_TXT}" ]; then + echo "${VERSIONS_TXT} not found. Run this from \`make sync-cpp\`." >&2 + exit 1 +fi + # Load `DEPENDENCIES` and the *_BUILD_SHA256_CHECKSUM variables. # shellcheck source=/dev/null -source "${SOURCE_DIR}/cpp/thirdparty/versions.txt" +source "${VERSIONS_TXT}" + +# Don't add macOS metadata (._* files, extended attributes) to the tarballs +export COPYFILE_DISABLE=1 sha256() { if command -v sha256sum > /dev/null 2>&1; then @@ -50,8 +100,150 @@ sha256() { fi } -mkdir -p "${DESTDIR}" -keep=() +# rm -rf the given paths, relative to the current directory +rmd() { + rm -rf "$@" +} + +# Remove everything in the current directory except the given names +keep_only() { + local entry name keep + for entry in * .[!.]*; do + [ -e "${entry}" ] || continue + keep=false + for name in "$@"; do + if [ "${entry}" = "${name}" ]; then + keep=true + fi + done + if [ "${keep}" = "false" ]; then + rm -rf "${entry}" + fi + done +} + +# Remove directories that no bundled build of our dependencies compiles. +# Arguments are paths (relative to the source root) to keep anyway. +trim_generic() { + local dir keep k + find . -depth -type d \( -name test -o -name tests -o -name testdata \ + -o -name doc -o -name docs -o -name example -o -name examples \ + -o -name benchmark -o -name benchmarks -o -name bench -o -name fuzz \) \ + -print | while read -r dir; do + keep=false + for k in "$@"; do + if [ "${dir}" = "./${k}" ]; then + keep=true + fi + done + if [ "${keep}" = "false" ]; then + rm -rf "${dir}" + fi + done +} + +# Trim an extracted source tree (the current directory) for the dependency +# named $1 (e.g. "thrift") +trim_dependency() { + case "$1" in + brotli) + rmd go python java csharp js research + # install() copies the man pages from docs/ + trim_generic docs + ;; + bzip2) + rm -f manual.* sample* ./*.html + trim_generic + ;; + lz4) + # Only lib/ is built (LZ4_BUILD_CLI=OFF). Files outside of lib/ are + # GPL-2.0-or-later, so don't ship the ones we don't need. + rmd programs contrib ossfuzz + trim_generic + (cd build && keep_only cmake) + ;; + mimalloc) + rmd ide + trim_generic + ;; + rapidjson) + rmd bin thirdparty + trim_generic + ;; + re2) + rmd benchlog python app + trim_generic + ;; + simdjson) + # Built from src/ and include/. singleheader/ is a 20 MB amalgamation. + rmd singleheader images jsonexamples tools scripts + trim_generic + ;; + thrift) + # Only the C++ library is built (BUILD_COMPILER=OFF, WITH_*=OFF) + rmd tutorial test compiler contrib doc debian + (cd lib && keep_only cpp CMakeLists.txt) + trim_generic + ;; + zlib) + rmd contrib + # zlib's CMakeLists.txt builds test/example.c unless ZLIB_BUILD_EXAMPLES=OFF + trim_generic test + ;; + zstd) + # ZSTD_BUILD_PROGRAMS=OFF + rmd contrib zlibWrapper programs + trim_generic + (cd build && keep_only cmake) + ;; + *) + trim_generic + ;; + esac +} + +# Extract $1, trim it for dependency $3 and write the result to $2 +trim_tarball() { + local in="$1" out="$2" name="$3" + local work src + if [ -f "${SOURCE_DIR}/trim/${name}.sh" ]; then + bash "${SOURCE_DIR}/trim/${name}.sh" "${in}" "${out}" + return + fi + work="$(mktemp -d)" + # Use CMake's tar (libarchive) for .tar.bz2: tar -j needs a bzip2 binary, + # which e.g. the R CI images don't have. + case "${in}" in + *.tar.bz2) (cd "${work}" && cmake -E tar xf "${in}") ;; + *) tar -xf "${in}" -C "${work}" ;; + esac + # Every tarball we vendor has a single top-level directory + src="$(cd "${work}" && ls)" + if [ "$(echo "${src}" | wc -l)" -ne 1 ] || [ ! -d "${work}/${src}" ]; then + echo "Expected a single top-level directory in ${in}" >&2 + rm -rf "${work}" + return 1 + fi + (cd "${work}/${src}" && trim_dependency "${name}") + case "${out}" in + *.tar.bz2) (cd "${work}" && cmake -E tar cjf "${out}" "${src}") ;; + *.tar.gz) tar -czf "${out}" -C "${work}" "${src}" ;; + *) + echo "Don't know how to compress ${out}" >&2 + rm -rf "${work}" + return 1 + ;; + esac + rm -rf "${work}" +} + +mkdir -p "${DESTDIR}" "${DOWNLOAD_DIR}" +# trim_tarball() changes directories, so it needs absolute paths +DOWNLOAD_DIR="$(cd "${DOWNLOAD_DIR}" && pwd)" +TRIM_DIR="$(mktemp -d)" +trap 'rm -rf "${TRIM_DIR}"' EXIT +keep=(checksums.txt) +sums="filename upstream_sha256 vendored_sha256"$'\n' for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do url="" for dep_packed in "${DEPENDENCIES[@]}"; do @@ -67,28 +259,38 @@ for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do echo "${vendored_url_var} not found in versions.txt" >&2 exit 1 fi - # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM + # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM and xsimd checksum_var="${vendored_url_var%_URL}_BUILD_SHA256_CHECKSUM" checksum="${!checksum_var}" + name="$(echo "${vendored_url_var}" | sed -e 's/^ARROW_//' -e 's/_URL$//' | tr '[:upper:]' '[:lower:]')" keep+=("${tar_name}") - out="${DESTDIR}/${tar_name}" - if [ -f "${out}" ] && [ "$(sha256 "${out}")" = "${checksum}" ]; then - continue + upstream="${DOWNLOAD_DIR}/${tar_name}" + if [ ! -f "${upstream}" ] || [ "$(sha256 "${upstream}")" != "${checksum}" ]; then + echo "Downloading ${url}" + curl --fail --silent --show-error --location --retry 3 --output "${upstream}" "${url}" + if [ "$(sha256 "${upstream}")" != "${checksum}" ]; then + echo "Checksum mismatch for ${upstream}" >&2 + rm -f "${upstream}" + exit 1 + fi fi - echo "Downloading ${url}" - curl --fail --silent --show-error --location --retry 3 --output "${out}" "${url}" - - if [ "$(sha256 "${out}")" != "${checksum}" ]; then - echo "Checksum mismatch for ${out}" >&2 - rm -f "${out}" - exit 1 - fi + # Trim into a temporary directory (keeping the file name, so the extension + # says how to compress it) so that a failure doesn't leave a partial file. + echo "Trimming ${tar_name}" + trim_tarball "${upstream}" "${TRIM_DIR}/${tar_name}" "${name}" + out="${DESTDIR}/${tar_name}" + mv "${TRIM_DIR}/${tar_name}" "${out}" + sums="${sums}${tar_name} ${checksum} $(sha256 "${out}")"$'\n' done -# Remove tarballs for other versions +# See read_vendored_checksums() in nixlibs.R +printf "%s" "${sums}" > "${DESTDIR}/checksums.txt" + +# Remove tarballs for other versions (or dependencies we no longer vendor) for file in "${DESTDIR}"/*; do + [ -e "${file}" ] || continue stale=true for name in "${keep[@]}"; do if [ "$(basename "${file}")" = "${name}" ]; then From 1947743395ad4cf6cf778d42bc30296fc4631cde Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 18:41:52 -0600 Subject: [PATCH 2/2] GH-55478: [C++][R] Build Thrift with header-only Boost for the R package Vendoring all of the Boost libraries that the bundled Boost build uses adds about 5 MB to the R package, even though Thrift (the only user of Boost in a non-test build) only needs a handful of headers. When Boost is only needed for Thrift (ARROW_BOOST_FOR_THRIFT_ONLY: no tests, Gandiva or other Boost users, native 128-bit integers, threading, not Windows), build_boost_headers_for_thrift() uses the Boost headers without running Boost's CMake build, and add_thrift_minimal_library() compiles only the Thrift sources that Parquet needs. Other builds use the existing path unchanged. This lets the R package vendor a Boost tarball of ~120 KB (the headers reachable from Thrift) and a Thrift tarball of ~220 KB (only the C++ library sources), trimmed by tools/trim/boost.sh and tools/trim/thrift.sh. Co-Authored-By: Claude Opus 5.5 --- cpp/cmake_modules/ThirdpartyToolchain.cmake | 270 ++++++++++++++++---- r/.Rbuildignore | 1 + r/tools/trim/boost.sh | 130 ++++++++++ r/tools/trim/thrift.sh | 54 ++++ 4 files changed, 400 insertions(+), 55 deletions(-) create mode 100644 r/tools/trim/boost.sh create mode 100644 r/tools/trim/thrift.sh diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 123def5b66c5..d067675b41ed 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1085,7 +1085,63 @@ endif() # ---------------------------------------------------------------------- # Add Boost dependencies (code adapted from Apache Kudu) +# Use the bundled Boost source archive header-only, without Boost's +# CMake build. This only provides the Boost libraries that Apache +# Thrift needs. See ARROW_BOOST_FOR_THRIFT_ONLY. +# +# This is a macro to set variables in build_boost()'s PARENT_SCOPE. +macro(build_boost_headers_for_thrift) + list(APPEND CMAKE_MESSAGE_INDENT "Boost: ") + message(STATUS "Using headers from source (only for Apache Thrift)") + + fetchcontent_declare(boost + ${FC_DECLARE_COMMON_OPTIONS} + URL ${BOOST_SOURCE_URL} + URL_HASH "SHA256=${ARROW_BOOST_BUILD_SHA256_CHECKSUM}" + # This doesn't exist. So FetchContent only + # extracts Boost and doesn't add Boost's CMake + # project. + SOURCE_SUBDIR "arrow-header-only") + prepare_fetchcontent() + fetchcontent_makeavailable(boost) + + # Boost.Uuid (used by thrift/TUuid.h) and the Boost libraries it + # includes headers from. If you change this, check + # r/tools/trim/boost.sh too. + set(libraries + assert + config + throw_exception + type_traits + uuid) + set(boost_include_dirs) + foreach(library ${libraries}) + set(include_dir "${boost_SOURCE_DIR}/libs/${library}/include") + if(NOT IS_DIRECTORY "${include_dir}") + message(FATAL_ERROR "Boost source archive doesn't have ${include_dir}") + endif() + list(APPEND boost_include_dirs "${include_dir}") + endforeach() + add_library(Boost::headers INTERFACE IMPORTED) + target_include_directories(Boost::headers INTERFACE ${boost_include_dirs}) + target_compile_definitions(Boost::headers INTERFACE BOOST_ALL_NO_LIB) + + set(Boost_INCLUDE_DIRS + ${boost_include_dirs} + PARENT_SCOPE) + set(BOOST_VENDORED + TRUE + PARENT_SCOPE) + + list(POP_BACK CMAKE_MESSAGE_INDENT) +endmacro() + function(build_boost) + if(ARROW_BOOST_FOR_THRIFT_ONLY) + build_boost_headers_for_thrift() + return() + endif() + list(APPEND CMAKE_MESSAGE_INDENT "Boost: ") message(STATUS "Building from source") @@ -1321,6 +1377,30 @@ else() set(ARROW_USE_BOOST FALSE) endif() +# If Boost is only needed because Apache Thrift's headers use it, we +# can avoid both Boost's and Apache Thrift's CMake builds: bundled +# Boost is used header-only (see build_boost_headers_for_thrift()) and +# bundled Apache Thrift is built from only the sources that Apache +# Parquet needs (see build_thrift()). This is faster and lets packagers +# (e.g. the R package, see r/tools/trim/boost.sh) ship a tiny subset of +# the Boost source archive. +# +# Other configurations keep using Boost's and Apache Thrift's own CMake +# builds. This is not used on Windows yet because the minimal Apache +# Thrift build hasn't been tested there. +if(ARROW_USE_BOOST + AND ARROW_WITH_THRIFT + AND NOT ARROW_BOOST_REQUIRE_LIBRARY + AND NOT ARROW_GANDIVA + AND NOT ARROW_TESTING + AND ARROW_USE_NATIVE_INT128 + AND ARROW_ENABLE_THREADING + AND NOT WIN32) + set(ARROW_BOOST_FOR_THRIFT_ONLY TRUE) +else() + set(ARROW_BOOST_FOR_THRIFT_ONLY FALSE) +endif() + if(ARROW_USE_BOOST) if(ARROW_BOOST_USE_SHARED) # Find shared Boost libraries. @@ -1790,6 +1870,82 @@ endif() # ---------------------------------------------------------------------- # Thrift +# Build only the parts of the Apache Thrift C++ library that Apache +# Parquet uses (TCompactProtocol and TMemoryBuffer), without Apache +# Thrift's CMake build. Apache Thrift's CMake build requires Boost's +# CMake package and builds many sources that need more of Boost +# (e.g. Boost.Locale for TJSONProtocol). See ARROW_BOOST_FOR_THRIFT_ONLY. +# +# This is a macro to be used in build_thrift(). +macro(add_thrift_minimal_library) + fetchcontent_declare(thrift + ${FC_DECLARE_COMMON_OPTIONS} + URL ${THRIFT_SOURCE_URL} + URL_HASH "SHA256=${ARROW_THRIFT_BUILD_SHA256_CHECKSUM}" + # Only extract. We don't use Apache Thrift's + # CMake build. + SOURCE_SUBDIR "arrow-no-cmake") + prepare_fetchcontent() + fetchcontent_makeavailable(thrift) + + set(THRIFT_CPP_SOURCE_DIR "${thrift_SOURCE_DIR}/lib/cpp/src") + # If you update Apache Thrift, check whether Apache Parquet needs + # more sources (undefined symbols when linking libparquet users). + add_library(thrift STATIC + "${THRIFT_CPP_SOURCE_DIR}/thrift/TApplicationException.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/TOutput.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/TUuid.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/protocol/TProtocol.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/transport/TBufferTransports.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/transport/TTransportException.cpp") + set_target_properties(thrift PROPERTIES UNITY_BUILD OFF) + # Third-party code: don't fail on warnings. + target_compile_options(thrift PRIVATE $<$>:-Wno-error>) + target_compile_definitions(thrift PUBLIC THRIFT_STATIC_DEFINE) + target_include_directories(thrift SYSTEM + PUBLIC $ + $) + if(BOOST_VENDORED) + target_link_libraries(thrift PUBLIC $) + else() + target_link_libraries(thrift PUBLIC Boost::headers) + endif() + if(TARGET Threads::Threads) + target_link_libraries(thrift PUBLIC Threads::Threads) + endif() + + # Generate thrift/config.h from Apache Thrift's template, like + # Apache Thrift's build/cmake/ConfigureChecks.cmake does, but only + # with the checks that matter for the sources above and the headers + # they (and Apache Parquet) include. + include(CheckCXXSourceCompiles) + include(CheckIncludeFileCXX) + include(CheckSymbolExists) + check_include_file_cxx(inttypes.h ARROW_THRIFT_HAVE_INTTYPES_H) + check_include_file_cxx(netinet/in.h ARROW_THRIFT_HAVE_NETINET_IN_H) + check_include_file_cxx(stdint.h ARROW_THRIFT_HAVE_STDINT_H) + check_include_file_cxx(sys/param.h ARROW_THRIFT_HAVE_SYS_PARAM_H) + check_symbol_exists(strerror_r "string.h" ARROW_THRIFT_HAVE_STRERROR_R) + # GNU strerror_r() returns char * instead of int. + check_cxx_source_compiles("#include +int main() { char b; char* a = strerror_r(0, &b, 0); static_cast(a); }" + ARROW_THRIFT_STRERROR_R_CHAR_P) + foreach(name + HAVE_INTTYPES_H + HAVE_NETINET_IN_H + HAVE_STDINT_H + HAVE_SYS_PARAM_H + HAVE_STRERROR_R + STRERROR_R_CHAR_P) + set(${name} ${ARROW_THRIFT_${name}}) + endforeach() + set(PACKAGE_NAME "thrift") + set(PACKAGE_VERSION "${ARROW_THRIFT_BUILD_VERSION}") + set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") + configure_file("${thrift_SOURCE_DIR}/build/cmake/config.h.in" + "${thrift_BINARY_DIR}/thrift/config.h") +endmacro() + function(build_thrift) list(APPEND CMAKE_MESSAGE_INDENT "Thrift: ") message(STATUS "Building from source") @@ -1797,66 +1953,70 @@ function(build_thrift) if(CMAKE_VERSION VERSION_LESS 3.26) message(FATAL_ERROR "Require CMake 3.26 or later for building bundled Apache Thrift") endif() - fetchcontent_declare(thrift - ${FC_DECLARE_COMMON_OPTIONS} - URL ${THRIFT_SOURCE_URL} - URL_HASH "SHA256=${ARROW_THRIFT_BUILD_SHA256_CHECKSUM}") + if(ARROW_BOOST_FOR_THRIFT_ONLY) + add_thrift_minimal_library() + else() + fetchcontent_declare(thrift + ${FC_DECLARE_COMMON_OPTIONS} + URL ${THRIFT_SOURCE_URL} + URL_HASH "SHA256=${ARROW_THRIFT_BUILD_SHA256_CHECKSUM}") - prepare_fetchcontent() - set(BUILD_COMPILER OFF) - set(BUILD_EXAMPLES OFF) - set(BUILD_TUTORIALS OFF) - set(CMAKE_UNITY_BUILD OFF) - set(WITH_AS3 OFF) - set(WITH_CPP ON) - set(WITH_C_GLIB OFF) - set(WITH_JAVA OFF) - set(WITH_JAVASCRIPT OFF) - set(WITH_LIBEVENT OFF) - if(MSVC) - if(ARROW_USE_STATIC_CRT) - set(WITH_MT ON) - else() - set(WITH_MT OFF) + prepare_fetchcontent() + set(BUILD_COMPILER OFF) + set(BUILD_EXAMPLES OFF) + set(BUILD_TUTORIALS OFF) + set(CMAKE_UNITY_BUILD OFF) + set(WITH_AS3 OFF) + set(WITH_CPP ON) + set(WITH_C_GLIB OFF) + set(WITH_JAVA OFF) + set(WITH_JAVASCRIPT OFF) + set(WITH_LIBEVENT OFF) + if(MSVC) + if(ARROW_USE_STATIC_CRT) + set(WITH_MT ON) + else() + set(WITH_MT OFF) + endif() endif() - endif() - set(WITH_NODEJS OFF) - set(WITH_OPENSSL OFF) - set(WITH_PYTHON OFF) - set(WITH_QT5 OFF) - set(WITH_ZLIB OFF) - - # Apache Thrift may change CMAKE_DEBUG_POSTFIX. So we'll restore the - # original CMAKE_DEBUG_POSTFIX later. - set(CMAKE_DEBUG_POSTFIX_KEEP ${CMAKE_DEBUG_POSTFIX}) - - # Remove Apache Arrow's CMAKE_MODULE_PATH to ensure using Apache - # Thrift's cmake_modules/. - # - # We can remove this once https://github.com/apache/thrift/pull/3176 - # is merged. - list(POP_FRONT CMAKE_MODULE_PATH) - fetchcontent_makeavailable(thrift) + set(WITH_NODEJS OFF) + set(WITH_OPENSSL OFF) + set(WITH_PYTHON OFF) + set(WITH_QT5 OFF) + set(WITH_ZLIB OFF) + + # Apache Thrift may change CMAKE_DEBUG_POSTFIX. So we'll restore the + # original CMAKE_DEBUG_POSTFIX later. + set(CMAKE_DEBUG_POSTFIX_KEEP ${CMAKE_DEBUG_POSTFIX}) + + # Remove Apache Arrow's CMAKE_MODULE_PATH to ensure using Apache + # Thrift's cmake_modules/. + # + # We can remove this once https://github.com/apache/thrift/pull/3176 + # is merged. + list(POP_FRONT CMAKE_MODULE_PATH) + fetchcontent_makeavailable(thrift) - # Apache Thrift may change CMAKE_DEBUG_POSTFIX. So we restore - # CMAKE_DEBUG_POSTFIX. - set(CMAKE_DEBUG_POSTFIX - ${CMAKE_DEBUG_POSTFIX_KEEP} - CACHE BOOL "" FORCE) + # Apache Thrift may change CMAKE_DEBUG_POSTFIX. So we restore + # CMAKE_DEBUG_POSTFIX. + set(CMAKE_DEBUG_POSTFIX + ${CMAKE_DEBUG_POSTFIX_KEEP} + CACHE BOOL "" FORCE) - if(CMAKE_VERSION VERSION_LESS 3.28) - set_property(DIRECTORY ${thrift_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL TRUE) - endif() + if(CMAKE_VERSION VERSION_LESS 3.28) + set_property(DIRECTORY ${thrift_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL TRUE) + endif() - target_include_directories(thrift - INTERFACE $ - $ - ) - if(BOOST_VENDORED) - target_link_libraries(thrift PUBLIC $) - target_link_libraries(thrift PRIVATE $) - else() - target_link_libraries(thrift INTERFACE Boost::headers) + target_include_directories(thrift + INTERFACE $ + $ + ) + if(BOOST_VENDORED) + target_link_libraries(thrift PUBLIC $) + target_link_libraries(thrift PRIVATE $) + else() + target_link_libraries(thrift INTERFACE Boost::headers) + endif() endif() add_library(thrift::thrift INTERFACE IMPORTED) diff --git a/r/.Rbuildignore b/r/.Rbuildignore index 698ccf3f7afc..faae1aaa4480 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -33,3 +33,4 @@ STYLE.md ^bootstrap.R$ air.toml ^tools/vendored-downloads$ +^tools/trim$ diff --git a/r/tools/trim/boost.sh b/r/tools/trim/boost.sh new file mode 100644 index 000000000000..bbbeba6dd566 --- /dev/null +++ b/r/tools/trim/boost.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Trim the Boost source archive (boost-X.Y.Z-cmake.tar.gz) to the +# headers that Apache Thrift needs. +# +# The R package only needs Boost for Apache Thrift (for Parquet), and +# Arrow's CMake uses bundled Boost header-only in that case (see +# ARROW_BOOST_FOR_THRIFT_ONLY in cpp/cmake_modules/ThirdpartyToolchain.cmake). +# So we keep only the headers reachable via #include from the Boost +# headers that Apache Thrift includes, and keep the directory layout of +# the upstream archive (boost-X.Y.Z/libs//include/boost/...). +# +# The #include graph is followed without evaluating the preprocessor, +# so headers for all platforms and compilers are kept. boost/config is +# kept entirely because it includes headers via macros. +# +# Usage: tools/trim/boost.sh IN_TARBALL OUT_TARBALL + +set -euo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2 + exit 1 +fi + +# The Boost headers that Apache Thrift's C++ library includes (in the +# sources Arrow builds and in the headers Parquet includes). Update +# this when Apache Thrift is upgraded: +# grep -rh '#include /dev/null | grep -q "GNU tar"; then + wildcards="--wildcards" +fi +tar -xzf "${in}" -C "${work}/src" ${wildcards} \ + '*/libs/*/include/boost/*' \ + '*/LICENSE_1_0.txt' +top="$(ls "${work}/src")" +cd "${work}/src/${top}" + +find libs -type f -path '*/include/boost/*' > "${work}/index" + +# Breadth-first search of the #include graph. Input: the index of all +# headers. Output: the headers to keep. +awk -v seeds="${SEEDS}" ' +function normalize(path, n, parts, i, out, m, stack) { + n = split(path, parts, "/") + m = 0 + for (i = 1; i <= n; i++) { + if (parts[i] == "" || parts[i] == ".") continue + if (parts[i] == "..") { if (m > 0) m--; continue } + stack[++m] = parts[i] + } + out = "" + for (i = 1; i <= m; i++) out = out (i > 1 ? "/" : "") stack[i] + return out +} +{ + # libs/uuid/include/boost/uuid/uuid.hpp -> boost/uuid/uuid.hpp + header = substr($0, index($0, "/include/boost/") + length("/include/")) + path[header] = $0 +} +END { + tail = split(seeds, queue, " ") + head = 0 + while (head < tail) { + header = queue[++head] + if ((header in seen) || !(header in path)) continue + seen[header] = 1 + print path[header] + dir = header + sub(/\/[^\/]*$/, "", dir) + file = path[header] + while ((getline line < file) > 0) { + if (line !~ /^[ \t]*#[ \t]*include[ \t]*[<"]/) continue + sub(/^[ \t]*#[ \t]*include[ \t]*[<"]/, "", line) + sub(/[>"].*$/, "", line) + if (line ~ /^boost\//) { + queue[++tail] = line + } else { + relative = normalize(dir "/" line) + if (relative in path) queue[++tail] = relative + } + } + close(file) + } +}' "${work}/index" > "${work}/keep" + +if ! grep -q "/uuid/uuid.hpp$" "${work}/keep"; then + echo "Failed to find Boost headers for Apache Thrift in ${in}" >&2 + exit 1 +fi + +{ + cat "${work}/keep" + find libs/config/include -type f + echo LICENSE_1_0.txt +} | LC_ALL=C sort -u | sed -e "s|^|${top}/|" > "${work}/files" + +echo "Keeping $(wc -l < "${work}/files" | tr -d ' ') files from Boost libraries:" \ + "$(sed -n -e 's|^[^/]*/libs/\([^/]*\)/.*|\1|p' "${work}/files" | sort -u | tr '\n' ' ')" + +cd "${work}/src" +tar -cf - -T "${work}/files" | gzip -n -9 > "${out}" diff --git a/r/tools/trim/thrift.sh b/r/tools/trim/thrift.sh new file mode 100644 index 000000000000..b71df2fdad13 --- /dev/null +++ b/r/tools/trim/thrift.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Trim the Apache Thrift source archive to what Arrow's minimal +# Apache Thrift build uses: the C++ library sources and the config.h +# template. Arrow doesn't use Apache Thrift's CMake build when Boost is +# only needed for Apache Thrift (see ARROW_BOOST_FOR_THRIFT_ONLY and +# add_thrift_minimal_library() in +# cpp/cmake_modules/ThirdpartyToolchain.cmake), which is always the case +# for the R package. +# +# Usage: tools/trim/thrift.sh IN_TARBALL OUT_TARBALL + +set -euo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2 + exit 1 +fi + +in="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +out="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" + +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT +tar -xzf "${in}" -C "${work}" +top="$(ls "${work}")" + +cd "${work}" +for path in LICENSE NOTICE build/cmake/config.h.in lib/cpp/src/thrift; do + if [ ! -e "${top}/${path}" ]; then + echo "${top}/${path} not found in ${in}" >&2 + exit 1 + fi +done +find "${top}/LICENSE" "${top}/NOTICE" "${top}/build/cmake/config.h.in" \ + "${top}/lib/cpp/src/thrift" -type f | LC_ALL=C sort > files +tar -cf - -T files | gzip -n -9 > "${out}"