diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh
index 20e6062e847f..ac9fbf2c479d 100755
--- a/ci/scripts/r_test.sh
+++ b/ci/scripts/r_test.sh
@@ -158,6 +158,26 @@ if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then
cp /etc/resolv.conf{.bak,}
fi
+if [ -n "${ARROW_R_EXPECTED_FEATURES:-}" ]; then
+ # The tests skip what the build doesn't support, so make sure the features
+ # we expect (e.g. from the vendored thirdparty sources) were actually built.
+ # ARROW_R_EXPECTED_FEATURES is a comma-separated list of
+ # names(arrow::arrow_info()$capabilities) and memory pool backends
+ # (e.g. mimalloc).
+ ${R_BIN} -s -e "
+ library(arrow, lib.loc = 'check/arrow.Rcheck')
+ info <- arrow_info()
+ print(info\$capabilities)
+ print(info\$memory_pool\$available_backends)
+ available <- c(names(which(info\$capabilities)), info\$memory_pool\$available_backends)
+ expected <- strsplit(Sys.getenv('ARROW_R_EXPECTED_FEATURES'), ',')[[1]]
+ missing <- setdiff(expected, available)
+ if (length(missing) > 0) {
+ stop('Expected features are not available: ', paste(missing, collapse = ', '))
+ }
+ "
+fi
+
AFTER=$(ls -alh ~/)
if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then
# Ignore ~/.TinyTex/ and ~/R/ because it has many files.
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index 3c5518a9596e..ef6e1a4f88dd 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -97,6 +97,7 @@ r/tests/testthat/_snaps/*.md
r/inst/include/cpp11.hpp
r/inst/include/cpp11/*.hpp
r/tools/nixlibs-allowlist.txt
+r/inst/COPYRIGHTS
.gitattributes
ruby/red-arrow/.yardopts
ruby/red-arrow-format/lib/arrow-format/org/*
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index d9351d3afe0a..8ea23b6d93b4 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -633,14 +633,30 @@ tasks:
template: r/github.linux.offline.build.yml
# Simulate a CRAN builder without network access: the build should detect
- # that it is offline and fall back to a minimal build without downloading
- # anything.
- test-r-offline-minimal:
- ci: github
- template: docker-tests/github.linux.yml
- params:
- flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true'
+ # that it is offline and build libarrow without downloading anything, using
+ # the thirdparty sources vendored in the package (tools/vendored) to still
+ # include Parquet, Dataset, JSON, the compression codecs, etc.
+ # ARROW_SOURCE_HOME='' makes it use the C++ source copied into tools/cpp
+ # (like the CRAN package does), whose versions.txt has the checksums of the
+ # trimmed tarballs. The codecs that are off by default on Linux are turned on
+ # so that every vendored tarball is built.
+ test-r-offline-vendored:
+ ci: github
+ template: docker-tests/github.linux.yml
+ params:
+ flags: >-
+ -e ARROW_R_TEST_OFFLINE=true
+ -e NOT_CRAN=false
+ -e SKIP_VIGNETTES=true
+ -e ARROW_SOURCE_HOME=''
+ -e ARROW_WITH_BROTLI=ON
+ -e ARROW_WITH_BZ2=ON
+ -e ARROW_WITH_ZSTD=ON
+ -e ARROW_R_EXPECTED_FEATURES=dataset,parquet,json,mimalloc,re2,utf8proc,snappy,gzip,brotli,zstd,lz4,bz2
image: r
+ # A full libarrow build with 2 cores (NOT_CRAN=false) and no sccache
+ # (it's offline)
+ timeout: 180
# This one has -flto=auto
test-r-ubuntu-22.04:
diff --git a/r/.Rbuildignore b/r/.Rbuildignore
index deaf5dd3948c..faae1aaa4480 100644
--- a/r/.Rbuildignore
+++ b/r/.Rbuildignore
@@ -32,3 +32,5 @@ STYLE.md
^inst/__pycache__$
^bootstrap.R$
air.toml
+^tools/vendored-downloads$
+^tools/trim$
diff --git a/r/.gitignore b/r/.gitignore
index a1295ea4e0b5..e16bc75f459b 100644
--- a/r/.gitignore
+++ b/r/.gitignore
@@ -35,3 +35,5 @@ extra-tests/files
# Source tarballs of required dependencies that we ship for offline builds.
# They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh).
/tools/vendored/
+# Cache of the upstream (untrimmed) tarballs used by tools/vendor_dependencies.sh
+/tools/vendored-downloads/
diff --git a/r/DESCRIPTION b/r/DESCRIPTION
index 1e5733c37cd5..19918beab685 100644
--- a/r/DESCRIPTION
+++ b/r/DESCRIPTION
@@ -24,6 +24,7 @@ Description: 'Apache' 'Arrow' is a cross-language
package provides an interface to the 'Arrow C++' library.
Depends: R (>= 4.2)
License: Apache License (>= 2.0)
+Copyright: See file inst/COPYRIGHTS
URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/
BugReports: https://github.com/apache/arrow/issues
Encoding: UTF-8
diff --git a/r/inst/COPYRIGHTS b/r/inst/COPYRIGHTS
new file mode 100644
index 000000000000..779b8b2a5fa8
--- /dev/null
+++ b/r/inst/COPYRIGHTS
@@ -0,0 +1,78 @@
+The arrow R package is Copyright The Apache Software Foundation and is licensed
+under the Apache License, Version 2.0. The Arrow C++ library source copied into
+tools/cpp includes code from other projects; see tools/LICENSE.txt and
+tools/NOTICE.txt in the source package.
+
+To build the Arrow C++ library without network access, the source package also
+includes (trimmed) source tarballs of these thirdparty dependencies in
+tools/vendored. They are only used when the dependency isn't found on the
+system, and are statically linked into the package when used. Each tarball
+contains the dependency's full license text.
+
+Boost (headers used by Apache Thrift)
+ Copyright the Boost authors
+ Boost Software License 1.0
+
+Brotli
+ Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors
+ MIT License
+
+bzip2
+ Copyright (C) 1996-2019 Julian R Seward
+ BSD-style license (see LICENSE in the tarball)
+
+LZ4 (lib/ only)
+ Copyright (c) 2011-2020, Yann Collet
+ BSD 2-Clause License
+
+mimalloc
+ Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen
+ MIT License
+
+RapidJSON
+ Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip
+ MIT License (with code under the BSD and JSON licenses, see license.txt in
+ the tarball)
+
+RE2
+ Copyright (c) 2009 The RE2 Authors
+ BSD 3-Clause License
+
+simdjson
+ Copyright 2018-2025 The simdjson authors
+ Apache License 2.0 or MIT License
+
+Snappy
+ Copyright 2011, Google Inc.
+ BSD 3-Clause License
+
+Apache Thrift
+ Copyright (C) 2006-2019, The Apache Software Foundation
+ Apache License 2.0
+
+uriparser (library sources only)
+ Copyright (C) 2007, Weijia Song and Sebastian Pipping
+ BSD 3-Clause License
+
+utf8proc
+ Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Tony Kelman,
+ Jonas Fonseca, and other contributors
+ Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany
+ MIT License, with Unicode data under the Unicode Data Files and Software
+ License
+
+xsimd
+ Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and
+ Martin Renou
+ Copyright (c) 2016, QuantStack
+ Copyright (c) 2018, Serge Guelton
+ BSD 3-Clause License
+
+zlib
+ Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler
+ zlib License
+
+Zstandard (zstd)
+ Copyright (c) Meta Platforms, Inc. and affiliates
+ BSD 3-Clause License (zstd is dual-licensed BSD/GPLv2; it is used under the
+ BSD license)
diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R
index 226894372ec5..a2c321647bd5 100644
--- a/r/tools/nixlibs.R
+++ b/r/tools/nixlibs.R
@@ -486,7 +486,7 @@ find_local_source <- function() {
# The second probably occurs if we're installing the arrow R package
cpp_dir_options <- c(
file.path(Sys.getenv("ARROW_SOURCE_HOME", ".."), "cpp"),
- "tools/cpp"
+ package_cpp_dir
)
for (cpp_dir in cpp_dir_options) {
if (file.exists(file.path(cpp_dir, "src/arrow/api.h"))) {
@@ -625,30 +625,35 @@ build_libarrow <- function(src_dir, dst_dir) {
env_var_list <- with_cloud_support(env_var_list)
env_var_list <- with_wasm_support(env_var_list)
- # turn_off_all_optional_features() needs to happen after
- # with_cloud_support(), since it might turn features ON.
+ # turn_off_all_optional_features() and turn_off_unvendored_features() need to
+ # happen after with_cloud_support(), since it might turn features ON.
thirdparty_deps_unavailable <- !download_ok &&
!dir.exists(thirdparty_dependency_dir) &&
!env_is("ARROW_DEPENDENCY_SOURCE", "system")
do_minimal_build <- env_is("LIBARROW_MINIMAL", "true")
+ # Thirdparty source tarballs shipped in the package
+ vendored <- usable_vendored_dependencies(src_dir)
if (do_minimal_build) {
env_var_list <- turn_off_all_optional_features(env_var_list)
} else if (thirdparty_deps_unavailable) {
cat(paste0(
"*** Building C++ library from source, but downloading thirdparty dependencies\n",
- " is not possible, so this build will turn off all thirdparty features.\n",
+ " is not possible, so this build will turn off the thirdparty features\n",
+ " whose sources are not included in the package.\n",
" See installation guide for details:\n",
" https://arrow.apache.org/docs/r/articles/install.html\n"
))
- env_var_list <- turn_off_all_optional_features(env_var_list)
+ env_var_list <- turn_off_unvendored_features(env_var_list, names(vendored))
} else if (dir.exists(thirdparty_dependency_dir)) {
# Add the *_SOURCE_URL env vars
env_var_list <- set_thirdparty_urls(env_var_list)
}
- # Some thirdparty dependencies are required even for a minimal build, so they
- # can't be turned off like the ones above.
- env_var_list <- with_vendored_dependencies(env_var_list)
+ # Point the build at the vendored sources of the thirdparty dependencies.
+ # Some of them are required even for a minimal build, so they can't be turned
+ # off like the ones above.
+ env_var_list <- with_vendored_dependencies(env_var_list, vendored)
+ use_vendored_checksums(src_dir, env_var_list, vendored)
env_vars <- env_vars_as_string(env_var_list)
lg("arrow %s", ifelse(quietly, "", paste("with", env_vars)), .indent = "****")
@@ -795,7 +800,9 @@ cmake_version <- function(cmd = "cmake") {
)
}
-turn_off_all_optional_features <- function(env_var_list) {
+# `keep` is a vector of features (names in `turn_off` below, or
+# "ARROW_SIMD_LEVEL") to leave as they are.
+turn_off_all_optional_features <- function(env_var_list, keep = character()) {
# Because these are done as environment variables (as opposed to build flags),
# setting these to "OFF" overrides any previous setting. We don't need to
# check the existing value.
@@ -822,19 +829,53 @@ turn_off_all_optional_features <- function(env_var_list) {
"ARROW_WITH_ZLIB" = "OFF",
"ARROW_WITH_ZSTD" = "OFF",
"ARROW_WITH_RE2" = "OFF",
- "ARROW_WITH_UTF8PROC" = "OFF",
+ "ARROW_WITH_UTF8PROC" = "OFF"
+ )
+ turn_off <- turn_off[setdiff(names(turn_off), keep)]
+ if (!("ARROW_SIMD_LEVEL" %in% keep)) {
# The syntax to turn off XSIMD is different.
# Pull existing value of EXTRA_CMAKE_FLAGS first (must be defined)
- "EXTRA_CMAKE_FLAGS" = paste(
- env_var_list[["EXTRA_CMAKE_FLAGS"]],
- "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE"
+ turn_off <- c(
+ turn_off,
+ "EXTRA_CMAKE_FLAGS" = paste(
+ env_var_list[["EXTRA_CMAKE_FLAGS"]],
+ "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE"
+ )
)
- )
+ }
# Create a new env_var_list, with the values of turn_off set.
# replace() also adds new values if they didn't exist before
replace(env_var_list, names(turn_off), turn_off)
}
+# Without network access, libarrow can only build the thirdparty dependencies
+# whose source tarballs are shipped in the package (see
+# `with_vendored_dependencies()`), so turn off every optional feature except
+# those whose dependencies are all vendored. Features that are kept are left as
+# they are (e.g. as the user or the platform defaults set them), not turned on.
+# `available` is the ARROW_*_URL names of the usable vendored tarballs (see
+# `usable_vendored_dependencies()`).
+turn_off_unvendored_features <- function(env_var_list, available) {
+ has_all <- vapply(vendored_features, function(deps) all(deps %in% available), logical(1))
+ keep <- names(vendored_features)[has_all]
+ if ("ARROW_PARQUET" %in% keep) {
+ # Dataset needs no thirdparty dependencies itself, but it's only useful
+ # with Parquet (and we've always turned them on and off together)
+ keep <- c(keep, "ARROW_DATASET")
+ }
+ if ("ARROW_XSIMD_URL" %in% available) {
+ keep <- c(keep, "ARROW_SIMD_LEVEL")
+ }
+ if (!quietly) {
+ lg(
+ "Features with vendored dependencies: %s",
+ if (length(keep)) paste(keep, collapse = ", ") else "none",
+ .indent = "****"
+ )
+ }
+ turn_off_all_optional_features(env_var_list, keep = keep)
+}
+
get_component_names <- function() {
if (!isTRUE(Sys.which("bash") != "")) {
stop("nixlibs.R requires bash to be installed and available in your PATH")
@@ -874,26 +915,53 @@ set_thirdparty_urls <- function(env_var_list) {
env_var_list
}
-# libarrow requires some thirdparty dependencies, even for a minimal build.
-# The libarrow build uses them from the system if they are available and
-# otherwise builds them from source, downloading from ARROW_*_URL. To make sure
-# that works without network access, `make sync-cpp` saves their source
-# tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here.
+# The libarrow build uses thirdparty dependencies from the system if they are
+# available and otherwise builds them from source, downloading from ARROW_*_URL.
+# Some are required even for a minimal build. To make sure that works without
+# network access, `make sync-cpp` saves (trimmed) source tarballs into
+# vendored_dependency_dir and we point ARROW_*_URL at them here.
# This does not change whether the system versions are used.
-with_vendored_dependencies <- function(env_var_list) {
- if (!dir.exists(vendored_dependency_dir)) {
+# `vendored` is the result of `usable_vendored_dependencies()`.
+with_vendored_dependencies <- function(env_var_list, vendored) {
+ for (env_varname in names(vendored)) {
+ if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) {
+ # Respect the user's (or set_thirdparty_urls()'s) choice
+ next
+ }
+ env_var_list <- replace(env_var_list, env_varname, vendored[[env_varname]])
+ }
+ env_var_list
+}
+
+# Returns the paths of the vendored tarballs that the libarrow source in
+# `src_dir` can use, named by their ARROW_*_URL.
+# The tarballs are trimmed, so the libarrow build only accepts them once
+# `use_vendored_checksums()` has put their checksums in versions.txt. We only
+# do that to the package's own copy of the source (tools/cpp), not e.g. ../cpp
+# of a git checkout, and only for tarballs that were trimmed from the version
+# that versions.txt pins (according to the checksums.txt file that
+# tools/vendor_dependencies.sh writes).
+usable_vendored_dependencies <- function(src_dir) {
+ manifest <- read_vendored_checksums()
+ if (is.null(manifest)) {
# e.g. building from a git checkout without `make sync-cpp`
if (!quietly) {
lg("No vendored dependencies found, they will be built with CMake if needed", .indent = "****")
}
- return(env_var_list)
+ return(character())
}
+ is_package_cpp_dir <- !is.null(src_dir) &&
+ identical(normalizePath(src_dir, mustWork = FALSE), normalizePath(package_cpp_dir, mustWork = FALSE))
+ if (!is_package_cpp_dir) {
+ if (!quietly) {
+ lg("Not using vendored dependencies, they are only used to build %s", package_cpp_dir, .indent = "****")
+ }
+ return(character())
+ }
+ expected <- read_thirdparty_checksums(file.path(src_dir, "thirdparty", "versions.txt"))
deps_df <- get_component_names()
+ usable <- character()
for (env_varname in vendored_dependencies) {
- if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) {
- # Respect the user's (or set_thirdparty_urls()'s) choice
- next
- }
filename <- deps_df$filename[deps_df$env_varname == env_varname]
tarball <- file.path(vendored_dependency_dir, filename)
if (length(tarball) != 1 || !file.exists(tarball)) {
@@ -902,9 +970,68 @@ with_vendored_dependencies <- function(env_var_list) {
}
next
}
- env_var_list <- replace(env_var_list, env_varname, normalizePath(tarball))
+ checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname)
+ entry <- manifest[manifest$filename == filename, , drop = FALSE]
+ # versions.txt has the vendored checksum if use_vendored_checksums() has
+ # already run on it (e.g. when installing from the r/ directory again)
+ matches <- nrow(entry) == 1 &&
+ unname(expected[checksum_varname]) %in% c(entry$upstream_sha256, entry$vendored_sha256)
+ if (!matches) {
+ if (!quietly) {
+ lg(
+ "Vendored source for %s doesn't match %s, it will be downloaded if needed",
+ env_varname,
+ file.path(src_dir, "thirdparty", "versions.txt"),
+ .indent = "****"
+ )
+ }
+ next
+ }
+ usable[[env_varname]] <- normalizePath(tarball)
}
- env_var_list
+ usable
+}
+
+# Read the checksums.txt that tools/vendor_dependencies.sh writes, with the
+# checksums of each vendored tarball before (upstream) and after trimming
+read_vendored_checksums <- function() {
+ manifest_file <- file.path(vendored_dependency_dir, "checksums.txt")
+ if (!file.exists(manifest_file)) {
+ return(NULL)
+ }
+ read.table(manifest_file, header = TRUE, stringsAsFactors = FALSE)
+}
+
+# The libarrow build checks the checksums of the thirdparty tarballs against
+# src_dir's versions.txt, so replace the upstream checksums of the vendored
+# tarballs that `env_var_list` points at with the trimmed tarballs' checksums.
+use_vendored_checksums <- function(src_dir, env_var_list, vendored) {
+ used <- vendored[vapply(
+ names(vendored),
+ function(env_varname) identical(env_var_list[[env_varname]], vendored[[env_varname]]),
+ logical(1)
+ )]
+ if (length(used) == 0) {
+ return(invisible())
+ }
+ manifest <- read_vendored_checksums()
+ versions_txt <- file.path(src_dir, "thirdparty", "versions.txt")
+ lines <- readLines(versions_txt)
+ for (env_varname in names(used)) {
+ checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname)
+ checksum <- manifest$vendored_sha256[manifest$filename == basename(used[[env_varname]])]
+ lines <- sub(paste0("^", checksum_varname, "=.*$"), paste0(checksum_varname, "=", checksum), lines)
+ }
+ writeLines(lines, versions_txt)
+}
+
+# Parse the *_BUILD_SHA256_CHECKSUM=... lines of a versions.txt into a named vector
+read_thirdparty_checksums <- function(versions_txt) {
+ if (!file.exists(versions_txt)) {
+ return(character())
+ }
+ lines <- grep("^[A-Z0-9_]+_BUILD_SHA256_CHECKSUM=", readLines(versions_txt), value = TRUE)
+ setNames(sub("^[^=]*=", "", lines), sub("=.*$", "", lines))
}
# this is generally about features that people asked for via environment variables, but
@@ -1140,11 +1267,37 @@ if (download_libarrow_ok && !is_release && !test_mode) {
# `create_package_with_all_dependencies()` in install-arrow.R
thirdparty_dependency_dir <- Sys.getenv("ARROW_THIRDPARTY_DEPENDENCY_DIR", "tools/thirdparty_dependencies")
-# Source tarballs of dependencies that libarrow requires even for a minimal build.
-# These are always shipped in the package (see `with_vendored_dependencies()`).
-# If you change this path or list, you also need to edit tools/vendor_dependencies.sh
+# Trimmed source tarballs of thirdparty dependencies, shipped in the package so
+# that building without network access still works and can include the features
+# below (see `with_vendored_dependencies()` and `turn_off_unvendored_features()`).
+# If you change this path or these lists, you also need to edit
+# tools/vendor_dependencies.sh
vendored_dependency_dir <- "tools/vendored"
-vendored_dependencies <- c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL")
+# The copy of the libarrow source in the package (see `make sync-cpp`)
+package_cpp_dir <- "tools/cpp"
+# Optional features and the ARROW_*_URL of each dependency they need (when the
+# dependency is built from source). See "Some EP's require other EP's" in
+# cpp/cmake_modules/ThirdpartyToolchain.cmake
+vendored_features <- list(
+ # Parquet turns on Thrift and simdjson; Thrift needs zlib and Boost
+ ARROW_PARQUET = c("ARROW_BOOST_URL", "ARROW_SIMDJSON_URL", "ARROW_THRIFT_URL", "ARROW_ZLIB_URL"),
+ ARROW_JSON = c("ARROW_RAPIDJSON_URL", "ARROW_SIMDJSON_URL"),
+ ARROW_MIMALLOC = "ARROW_MIMALLOC_URL",
+ ARROW_WITH_BROTLI = "ARROW_BROTLI_URL",
+ ARROW_WITH_BZ2 = "ARROW_BZIP2_URL",
+ ARROW_WITH_LZ4 = "ARROW_LZ4_URL",
+ ARROW_WITH_RE2 = "ARROW_RE2_URL",
+ ARROW_WITH_SNAPPY = "ARROW_SNAPPY_URL",
+ ARROW_WITH_UTF8PROC = "ARROW_UTF8PROC_URL",
+ ARROW_WITH_ZLIB = "ARROW_ZLIB_URL",
+ ARROW_WITH_ZSTD = "ARROW_ZSTD_URL"
+)
+# uriparser and xsimd are required even for a minimal build
+vendored_dependencies <- unique(c(
+ "ARROW_URIPARSER_URL",
+ "ARROW_XSIMD_URL",
+ unlist(vendored_features, use.names = FALSE)
+))
arrow_versioned <- paste0("arrow-", VERSION)
# configure.win uses a different libarrow dir and the zip is already nested
diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R
index 9d8abccfbb5e..0320658d41d4 100644
--- a/r/tools/test-nixlibs.R
+++ b/r/tools/test-nixlibs.R
@@ -233,58 +233,213 @@ test_that("find_latest_nightly()", {
)
})
-test_that("with_vendored_dependencies()", {
- vendored_dir <- tempfile()
- dir.create(vendored_dir)
- on.exit(unlink(vendored_dir, recursive = TRUE), add = TRUE)
- original_dir <- nixlibs_env$vendored_dependency_dir
- original_get_component_names <- nixlibs_env$get_component_names
- original_quietly <- nixlibs_env$quietly
- on.exit(nixlibs_env$vendored_dependency_dir <- original_dir, add = TRUE)
- on.exit(nixlibs_env$get_component_names <- original_get_component_names, add = TRUE)
- on.exit(nixlibs_env$quietly <- original_quietly, add = TRUE)
- nixlibs_env$vendored_dependency_dir <- vendored_dir
- nixlibs_env$get_component_names <- function() {
- data.frame(
- env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL"),
- filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz")
+# Set up a fake vendored_dependency_dir and package_cpp_dir for the vendored
+# dependency tests. `vendored` is a named vector of filename = upstream
+# checksum in checksums.txt (the trimmed checksum is "-trimmed"),
+# `expected` is the checksums in the source dir's versions.txt, named by
+# ARROW_*_URL.
+local_vendored_dependencies <- function(vendored, expected, env = parent.frame()) {
+ vendored_dir <- withr::local_tempdir(.local_envir = env)
+ src_dir <- withr::local_tempdir(.local_envir = env)
+ for (filename in names(vendored)) {
+ file.create(file.path(vendored_dir, filename))
+ }
+ if (length(vendored)) {
+ writeLines(
+ c(
+ "filename upstream_sha256 vendored_sha256",
+ paste(names(vendored), vendored, paste0(vendored, "-trimmed"))
+ ),
+ file.path(vendored_dir, "checksums.txt")
)
}
- withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA))
+ dir.create(file.path(src_dir, "thirdparty"))
+ writeLines(
+ c(
+ "# a comment",
+ paste0(sub("_URL$", "_BUILD_SHA256_CHECKSUM", names(expected)), "=", expected)
+ ),
+ file.path(src_dir, "thirdparty", "versions.txt")
+ )
+ local_binding("vendored_dependency_dir", vendored_dir, env)
+ local_binding("package_cpp_dir", src_dir, env)
+ local_binding(
+ "get_component_names",
+ function() {
+ data.frame(
+ env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL", "ARROW_JEMALLOC_URL"),
+ filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz", "jemalloc-5.3.0.tar.bz2")
+ )
+ },
+ env
+ )
+ list(vendored_dir = vendored_dir, src_dir = src_dir)
+}
+
+# Temporarily replace a variable in nixlibs_env
+local_binding <- function(name, value, env = parent.frame()) {
+ original <- nixlibs_env[[name]]
+ assign(name, value, envir = nixlibs_env)
+ withr::defer(assign(name, original, envir = nixlibs_env), envir = env)
+}
- # Without the vendored tarballs, nothing is set
- nixlibs_env$quietly <- TRUE
- expect_silent(
- expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1"))
+test_that("usable_vendored_dependencies()", {
+ local_binding("quietly", TRUE)
+ # Nothing vendored
+ dirs <- local_vendored_dependencies(character(), c(ARROW_URIPARSER_URL = "aaa"))
+ expect_identical(usable_vendored_dependencies(dirs$src_dir), character())
+
+ dirs <- local_vendored_dependencies(
+ c(
+ "uriparser-1.0.2.tar.bz2" = "aaa",
+ "xsimd-14.3.0.tar.gz" = "bbb",
+ "zstd-1.5.7.tar.gz" = "ccc",
+ # Not in vendored_dependencies, so it isn't used
+ "jemalloc-5.3.0.tar.bz2" = "ddd"
+ ),
+ c(
+ ARROW_URIPARSER_URL = "aaa",
+ # A different version than the vendored one
+ ARROW_XSIMD_URL = "other",
+ # Already has the trimmed checksum (use_vendored_checksums() has run)
+ ARROW_ZSTD_URL = "ccc-trimmed",
+ ARROW_JEMALLOC_URL = "ddd"
+ )
+ )
+ expect_identical(
+ usable_vendored_dependencies(dirs$src_dir),
+ c(
+ ARROW_URIPARSER_URL = normalizePath(file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2")),
+ ARROW_ZSTD_URL = normalizePath(file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz"))
+ )
+ )
+ # Mismatches are only reported with verbose logging
+ local_binding("quietly", FALSE)
+ expect_output(
+ usable_vendored_dependencies(dirs$src_dir),
+ "Vendored source for ARROW_XSIMD_URL doesn't match"
+ )
+
+ # Only used to build package_cpp_dir (not e.g. ../cpp of a git checkout)
+ other_src_dir <- withr::local_tempdir()
+ file.copy(file.path(dirs$src_dir, "thirdparty"), other_src_dir, recursive = TRUE)
+ expect_output(
+ expect_identical(usable_vendored_dependencies(other_src_dir), character()),
+ "Not using vendored dependencies"
+ )
+ expect_output(
+ expect_identical(usable_vendored_dependencies(NULL), character()),
+ "Not using vendored dependencies"
)
- # The missing tarballs are only reported with verbose logging
- nixlibs_env$quietly <- FALSE
+
+ # A missing tarball isn't used, even if it's in checksums.txt
+ unlink(file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz"))
expect_output(
- expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")),
- "Vendored source for ARROW_URIPARSER_URL not found"
+ expect_named(usable_vendored_dependencies(dirs$src_dir), "ARROW_URIPARSER_URL"),
+ "Vendored source for ARROW_ZSTD_URL not found"
)
- nixlibs_env$quietly <- TRUE
+})
- uriparser <- file.path(vendored_dir, "uriparser-1.0.2.tar.bz2")
- xsimd <- file.path(vendored_dir, "xsimd-14.3.0.tar.gz")
- # Only dependencies in vendored_dependencies are used
- zstd <- file.path(vendored_dir, "zstd-1.5.7.tar.gz")
- file.create(uriparser, xsimd, zstd)
+test_that("use_vendored_checksums()", {
+ local_binding("quietly", TRUE)
+ dirs <- local_vendored_dependencies(
+ c("uriparser-1.0.2.tar.bz2" = "aaa", "xsimd-14.3.0.tar.gz" = "bbb", "zstd-1.5.7.tar.gz" = "ccc"),
+ c(ARROW_URIPARSER_URL = "aaa", ARROW_XSIMD_URL = "bbb", ARROW_ZSTD_URL = "ccc", ARROW_JEMALLOC_URL = "ddd")
+ )
+ versions_txt <- file.path(dirs$src_dir, "thirdparty", "versions.txt")
+ original <- readLines(versions_txt)
+ vendored <- c(
+ ARROW_URIPARSER_URL = file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2"),
+ ARROW_XSIMD_URL = file.path(dirs$vendored_dir, "xsimd-14.3.0.tar.gz"),
+ ARROW_ZSTD_URL = file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz")
+ )
+
+ # Nothing is pointed at the vendored tarballs
+ use_vendored_checksums(dirs$src_dir, list(), vendored)
+ expect_identical(readLines(versions_txt), original)
+
+ # Only the ones that are (e.g. not when the user set ARROW_XSIMD_URL)
+ env_var_list <- list(
+ ARROW_URIPARSER_URL = vendored[["ARROW_URIPARSER_URL"]],
+ ARROW_XSIMD_URL = "https://example.com/xsimd.tar.gz",
+ ARROW_ZSTD_URL = vendored[["ARROW_ZSTD_URL"]]
+ )
+ use_vendored_checksums(dirs$src_dir, env_var_list, vendored)
expect_identical(
- with_vendored_dependencies(list(A = "1")),
- list(A = "1", ARROW_URIPARSER_URL = normalizePath(uriparser), ARROW_XSIMD_URL = normalizePath(xsimd))
+ readLines(versions_txt),
+ c(
+ "# a comment",
+ "ARROW_URIPARSER_BUILD_SHA256_CHECKSUM=aaa-trimmed",
+ "ARROW_XSIMD_BUILD_SHA256_CHECKSUM=bbb",
+ "ARROW_ZSTD_BUILD_SHA256_CHECKSUM=ccc-trimmed",
+ "ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=ddd"
+ )
+ )
+ # and they are still usable afterwards
+ expect_named(
+ usable_vendored_dependencies(dirs$src_dir),
+ c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL")
+ )
+})
+
+test_that("with_vendored_dependencies()", {
+ vendored <- c(ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz")
+ withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA))
+
+ expect_identical(with_vendored_dependencies(list(A = "1"), character()), list(A = "1"))
+ expect_identical(
+ with_vendored_dependencies(list(A = "1"), vendored),
+ list(A = "1", ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz")
)
# Already set (e.g. by set_thirdparty_urls()) is respected
expect_identical(
- with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2")),
- list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = normalizePath(xsimd))
+ with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2"), vendored),
+ list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz")
)
# As is a user-set env var
withr::local_envvar(c(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2"))
expect_identical(
- with_vendored_dependencies(list(A = "1")),
- list(A = "1", ARROW_XSIMD_URL = normalizePath(xsimd))
+ with_vendored_dependencies(list(A = "1"), vendored),
+ list(A = "1", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz")
+ )
+})
+
+test_that("turn_off_unvendored_features()", {
+ local_binding("quietly", TRUE)
+ env_var_list <- list(EXTRA_CMAKE_FLAGS = "", ARROW_WITH_ZSTD = "ON", ARROW_S3 = "ON")
+
+ # Nothing vendored: the same as turn_off_all_optional_features()
+ expect_identical(
+ turn_off_unvendored_features(env_var_list, character()),
+ turn_off_all_optional_features(env_var_list)
+ )
+
+ # Everything vendored: only features that need unvendored dependencies are
+ # turned off, and the rest are left as they were (not turned on)
+ result <- turn_off_unvendored_features(env_var_list, vendored_dependencies)
+ expect_identical(result$ARROW_S3, "OFF")
+ expect_identical(result$ARROW_JEMALLOC, "OFF")
+ expect_identical(result$ARROW_WITH_ZSTD, "ON")
+ for (feature in setdiff(c(names(vendored_features), "ARROW_DATASET"), "ARROW_WITH_ZSTD")) {
+ expect_null(result[[feature]])
+ }
+ expect_identical(result$EXTRA_CMAKE_FLAGS, "")
+
+ # Parquet (and Dataset) need all of their dependencies
+ result <- turn_off_unvendored_features(
+ env_var_list,
+ c("ARROW_XSIMD_URL", "ARROW_THRIFT_URL", "ARROW_ZSTD_URL")
+ )
+ expect_identical(result$ARROW_PARQUET, "OFF")
+ expect_identical(result$ARROW_DATASET, "OFF")
+ expect_identical(result$ARROW_WITH_ZSTD, "ON")
+ expect_identical(result$ARROW_WITH_SNAPPY, "OFF")
+ # SIMD is only turned off if xsimd isn't vendored
+ expect_identical(result$EXTRA_CMAKE_FLAGS, "")
+ expect_match(
+ turn_off_unvendored_features(env_var_list, "ARROW_ZSTD_URL")$EXTRA_CMAKE_FLAGS,
+ "-DARROW_SIMD_LEVEL=NONE"
)
})
diff --git a/r/tools/trim/boost.sh b/r/tools/trim/boost.sh
new file mode 100644
index 000000000000..897da2e76b53
--- /dev/null
+++ b/r/tools/trim/boost.sh
@@ -0,0 +1,205 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Trim the Boost CMake source tarball (boost-X.Y.Z-cmake.tar.gz, ~140 MB)
+# down to the Boost libraries that libarrow's bundled Boost build
+# (build_boost() in cpp/cmake_modules/ThirdpartyToolchain.cmake) uses when
+# the R package builds libarrow, so it is small enough to ship in the R
+# package (~5 MB).
+#
+# Usage: tools/trim/boost.sh IN_TARBALL OUT_TARBALL
+#
+# Called by tools/vendor_dependencies.sh (`make sync-cpp`) after tools/cpp
+# has been synced. Needs cmake and a C++ compiler, because Boost's own CMake
+# is used to work out which libraries are needed.
+#
+# How it works:
+#
+# 1. Read the libraries build_boost() adds to BOOST_INCLUDE_LIBRARIES from
+# ThirdpartyToolchain.cmake, minus the ones that the R build never needs
+# (EXCLUDED_LIBRARIES below).
+# 2. Configure Boost with that BOOST_INCLUDE_LIBRARIES. Boost's CMake adds
+# the transitive dependencies and logs each library it adds.
+# 3. Keep only the top-level files, tools/cmake and those libraries, without
+# their tests, docs, examples, etc.
+#
+# Set BOOST_INCLUDE_LIBRARIES (semicolon-separated) in the environment to
+# skip step 1 and use that list instead.
+
+set -euo pipefail
+
+if [ "$#" -ne 2 ]; then
+ echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2
+ exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+IN="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
+OUT="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")"
+
+# Libraries that build_boost() can add to BOOST_INCLUDE_LIBRARIES but that
+# the R package's libarrow build never needs. If you add a library here,
+# say why.
+#
+# - multiprecision: only needed when ARROW_BOOST_NEED_MULTIPRECISION is
+# true, i.e. with ARROW_TESTING, ARROW_GANDIVA, ARROW_USE_UBSAN or a
+# compiler without a native 128-bit integer (ARROW_USE_NATIVE_INT128).
+# The R package builds none of those, and gcc/clang on the 64-bit
+# platforms that nixlibs.R builds for all have __int128 (32-bit targets
+# don't). Including it would more than double the size of the trimmed
+# tarball (~5 MB, 54 libraries -> ~11 MB, 58 libraries). If a build that
+# needs it uses this tarball, configuring libarrow fails in build_boost()
+# because the boost_multiprecision target doesn't exist.
+# - beast, variant, xpressive: only for ARROW_FLIGHT_SQL_ODBC.
+EXCLUDED_LIBRARIES=(multiprecision beast variant xpressive)
+
+# Print the library names that build_boost() adds to BOOST_INCLUDE_LIBRARIES
+# with `set(BOOST_INCLUDE_LIBRARIES ...)` or
+# `list(APPEND BOOST_INCLUDE_LIBRARIES ...)`, one per line. Variable
+# references such as ${ARROW_BOOST_COMPONENTS} are skipped: those are only
+# non-empty when building tests, integration tests, benchmarks or ODBC.
+boost_libraries_from_cmake() {
+ awk '
+ /^function\(build_boost\)/ { inside = 1; next }
+ inside && /^endfunction\(\)/ { exit }
+ inside {
+ sub(/#.*/, "")
+ gsub(/\(/, " ( ")
+ gsub(/\)/, " ) ")
+ for (i = 1; i <= NF; i++) tok[++n] = $i
+ }
+ END {
+ for (i = 1; i < n; i++) {
+ if (tok[i + 1] != "(") continue
+ cmd = tolower(tok[i])
+ j = i + 2
+ if (cmd == "set" && tok[j] == "BOOST_INCLUDE_LIBRARIES") {
+ j += 1
+ } else if (cmd == "list" && tok[j] == "APPEND" &&
+ tok[j + 1] == "BOOST_INCLUDE_LIBRARIES") {
+ j += 2
+ } else {
+ continue
+ }
+ for (; j <= n && tok[j] != ")"; j++) {
+ if (tok[j] !~ /^\$/) print tok[j]
+ }
+ }
+ }' "$1"
+}
+
+if [ -z "${BOOST_INCLUDE_LIBRARIES:-}" ]; then
+ # tools/cpp is the synced copy of cpp/ (`make sync-cpp`); fall back to the
+ # repository's cpp/ so this can also be run by hand from a git checkout.
+ TOOLCHAIN=""
+ for dir in "${SCRIPT_DIR}/../cpp" "${SCRIPT_DIR}/../../../cpp"; do
+ if [ -f "${dir}/cmake_modules/ThirdpartyToolchain.cmake" ]; then
+ TOOLCHAIN="${dir}/cmake_modules/ThirdpartyToolchain.cmake"
+ break
+ fi
+ done
+ if [ -z "${TOOLCHAIN}" ]; then
+ echo "Can't find ThirdpartyToolchain.cmake; run \`make sync-cpp\` first" >&2
+ exit 1
+ fi
+
+ ALL_LIBRARIES="$(boost_libraries_from_cmake "${TOOLCHAIN}")"
+ if [ -z "${ALL_LIBRARIES}" ]; then
+ echo "No BOOST_INCLUDE_LIBRARIES found in build_boost() in ${TOOLCHAIN}." >&2
+ echo "Has build_boost() changed? Update ${BASH_SOURCE[0]}." >&2
+ exit 1
+ fi
+ for lib in "${EXCLUDED_LIBRARIES[@]}"; do
+ if ! printf '%s\n' "${ALL_LIBRARIES}" | grep -qxF "${lib}"; then
+ echo "Warning: '${lib}' is in EXCLUDED_LIBRARIES but build_boost() no" \
+ "longer uses it; remove it from ${BASH_SOURCE[0]}" >&2
+ fi
+ done
+ BOOST_INCLUDE_LIBRARIES="$(printf '%s\n' "${ALL_LIBRARIES}" |
+ grep -vxF "$(printf '%s\n' "${EXCLUDED_LIBRARIES[@]}")" |
+ paste -s -d ';' -)"
+fi
+echo "Trimming Boost to BOOST_INCLUDE_LIBRARIES=${BOOST_INCLUDE_LIBRARIES} and their dependencies"
+
+WORK="$(mktemp -d "${TMPDIR:-/tmp}/arrow-trim-boost.XXXXXX")"
+trap 'rm -rf "${WORK}"' EXIT
+
+mkdir "${WORK}/src" "${WORK}/trimmed"
+tar -xzf "${IN}" -C "${WORK}/src"
+# The tarball has a single top-level directory, e.g. boost-1.92.0
+SRC_NAME="$(ls "${WORK}/src")"
+SRC="${WORK}/src/${SRC_NAME}"
+if [ ! -f "${SRC}/tools/cmake/include/BoostRoot.cmake" ]; then
+ echo "${IN} doesn't look like a Boost CMake source tarball" >&2
+ exit 1
+fi
+
+# Let Boost's CMake resolve the transitive dependencies. With
+# --log-level=VERBOSE, BoostRoot.cmake logs "Adding Boost library 'x'" for
+# each requested library and "Adding Boost dependency 'x'" for each
+# dependency (including 'headers', which is always added).
+# BUILD_TESTING=OFF matches prepare_fetchcontent(), so test-only
+# dependencies are not pulled in.
+cmake -S "${SRC}" -B "${WORK}/build" \
+ --log-level=VERBOSE \
+ -DBUILD_TESTING=OFF \
+ -DBOOST_INCLUDE_LIBRARIES="${BOOST_INCLUDE_LIBRARIES}" \
+ > "${WORK}/cmake.log" 2>&1 || {
+ cat "${WORK}/cmake.log" >&2
+ echo "Configuring Boost failed" >&2
+ exit 1
+}
+if grep -q "has not been found" "${WORK}/cmake.log"; then
+ grep "has not been found" "${WORK}/cmake.log" >&2
+ echo "A library in BOOST_INCLUDE_LIBRARIES doesn't exist in ${SRC_NAME}" >&2
+ exit 1
+fi
+# (bash 3.2 on macOS has no mapfile)
+KEEP_LIBRARIES=()
+while IFS= read -r lib; do
+ KEEP_LIBRARIES+=("${lib}")
+done < <(sed -n "s/.*Adding Boost [a-z]* '\([^']*\)'.*/\1/p" "${WORK}/cmake.log" | LC_ALL=C sort -u)
+if [ "${#KEEP_LIBRARIES[@]}" -eq 0 ]; then
+ echo "Found no \"Adding Boost ...\" messages in Boost's CMake output." >&2
+ echo "Has BoostRoot.cmake changed? Update ${BASH_SOURCE[0]}." >&2
+ exit 1
+fi
+echo "Keeping ${#KEEP_LIBRARIES[@]} Boost libraries: ${KEEP_LIBRARIES[*]}"
+
+# Copy what the bundled build needs into ${WORK}/trimmed/${SRC_NAME}:
+# the top-level files (CMakeLists.txt, LICENSE_1_0.txt, ...), tools/cmake
+# (BoostRoot.cmake etc.) and the libraries, but not the top-level doc/,
+# more/, status/ or the other tools/.
+DEST="${WORK}/trimmed/${SRC_NAME}"
+mkdir -p "${DEST}/tools"
+find "${SRC}" -maxdepth 1 -type f -exec cp -p {} "${DEST}/" \;
+cp -pR "${SRC}/tools/cmake" "${DEST}/tools/"
+for lib in "${KEEP_LIBRARIES[@]}"; do
+ # lib can be a nested library such as numeric/conversion
+ mkdir -p "$(dirname "${DEST}/libs/${lib}")"
+ cp -pR "${SRC}/libs/${lib}" "${DEST}/libs/${lib}"
+ # Only remove these at the top level of each library: deeper down they
+ # can be header directories (e.g. include/boost/math/tools/).
+ for dir in test doc example examples bench benchmark perf performance meta tools; do
+ rm -rf "${DEST}/libs/${lib:?}/${dir}"
+ done
+done
+
+# COPYFILE_DISABLE stops macOS tar from adding AppleDouble (._*) files.
+(cd "${WORK}/trimmed" && COPYFILE_DISABLE=1 tar -czf "${OUT}" "${SRC_NAME}")
diff --git a/r/tools/vendor_dependencies.sh b/r/tools/vendor_dependencies.sh
index caa28bff5ce2..686ce981e286 100644
--- a/r/tools/vendor_dependencies.sh
+++ b/r/tools/vendor_dependencies.sh
@@ -17,30 +17,80 @@
# specific language governing permissions and limitations
# under the License.
-# libarrow requires some thirdparty dependencies, even for a minimal build.
-# This script downloads the source tarballs that libarrow would otherwise
-# download at build time, so that they can be shipped in the R package.
-# nixlibs.R uses them when the dependencies are not found on the system, so
-# that building without network access (e.g. on CRAN) still works.
+# libarrow downloads the source of its thirdparty dependencies at build time
+# when they aren't found on the system. This script downloads those source
+# tarballs ahead of time so that they can be shipped in the R package, which
+# lets building without network access (e.g. on CRAN) still include features
+# like Parquet. nixlibs.R uses them when the dependencies are not found on the
+# system, and turns off the features whose dependencies aren't vendored.
+#
+# To keep the package small, each tarball is trimmed down to what the libarrow
+# build uses (no tests, docs, other language bindings, ...). Trimmed tarballs
+# no longer match the checksums pinned in versions.txt, so this script records
+# the upstream and trimmed checksums of each tarball in DESTDIR/checksums.txt.
+# When building the package's copy of the libarrow source (tools/cpp), nixlibs.R
+# uses the tarballs whose upstream checksum matches versions.txt and replaces
+# it with the trimmed one.
+#
+# Trimming is done by, in order of preference:
+# 1. tools/trim/.sh IN_TARBALL OUT_TARBALL, if it exists, where
+# is the lowercase dependency name (e.g. ARROW_BOOST_URL -> boost). Use
+# this for dependencies that need more than a few `rm`s.
+# 2. Otherwise, trim_generic() plus the dependency-specific removals in
+# trim_dependency() below.
#
# Usage: tools/vendor_dependencies.sh [DESTDIR]
-# Must be run after tools/cpp has been synced (see `make sync-cpp`).
+# Must be run after tools/cpp has been synced (see `make sync-cpp`). Needs curl
+# and cmake (to handle .tar.bz2 without a bzip2 binary).
+#
+# Environment variables:
+# ARROW_VENDOR_DOWNLOAD_DIR: where to cache the upstream (untrimmed)
+# tarballs between runs (default: tools/vendored-downloads, which is
+# ignored by git and R CMD build).
set -euo pipefail
SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DESTDIR="${1:-${SOURCE_DIR}/vendored}"
+DOWNLOAD_DIR="${ARROW_VENDOR_DOWNLOAD_DIR:-${SOURCE_DIR}/vendored-downloads}"
+VERSIONS_TXT="${SOURCE_DIR}/cpp/thirdparty/versions.txt"
# If you change this list, you also need to edit
-# `vendored_dependencies` in nixlibs.R
+# `vendored_dependencies` (and `vendored_features`) in nixlibs.R
VENDORED_DEPENDENCIES=(
+ # Required even for a minimal build
ARROW_URIPARSER_URL
ARROW_XSIMD_URL
+ # Parquet (ARROW_PARQUET turns on Thrift and simdjson; Thrift needs zlib and
+ # Boost headers)
+ ARROW_BOOST_URL
+ ARROW_SIMDJSON_URL
+ ARROW_THRIFT_URL
+ ARROW_ZLIB_URL
+ # Compression codecs
+ ARROW_BROTLI_URL
+ ARROW_BZIP2_URL
+ ARROW_LZ4_URL
+ ARROW_SNAPPY_URL
+ ARROW_ZSTD_URL
+ # Compute (RE2, utf8proc), JSON (RapidJSON, simdjson) and the allocator
+ ARROW_MIMALLOC_URL
+ ARROW_RAPIDJSON_URL
+ ARROW_RE2_URL
+ ARROW_UTF8PROC_URL
)
+if [ ! -f "${VERSIONS_TXT}" ]; then
+ echo "${VERSIONS_TXT} not found. Run this from \`make sync-cpp\`." >&2
+ exit 1
+fi
+
# Load `DEPENDENCIES` and the *_BUILD_SHA256_CHECKSUM variables.
# shellcheck source=/dev/null
-source "${SOURCE_DIR}/cpp/thirdparty/versions.txt"
+source "${VERSIONS_TXT}"
+
+# Don't add macOS metadata (._* files, extended attributes) to the tarballs
+export COPYFILE_DISABLE=1
sha256() {
if command -v sha256sum > /dev/null 2>&1; then
@@ -50,8 +100,150 @@ sha256() {
fi
}
-mkdir -p "${DESTDIR}"
-keep=()
+# rm -rf the given paths, relative to the current directory
+rmd() {
+ rm -rf "$@"
+}
+
+# Remove everything in the current directory except the given names
+keep_only() {
+ local entry name keep
+ for entry in * .[!.]*; do
+ [ -e "${entry}" ] || continue
+ keep=false
+ for name in "$@"; do
+ if [ "${entry}" = "${name}" ]; then
+ keep=true
+ fi
+ done
+ if [ "${keep}" = "false" ]; then
+ rm -rf "${entry}"
+ fi
+ done
+}
+
+# Remove directories that no bundled build of our dependencies compiles.
+# Arguments are paths (relative to the source root) to keep anyway.
+trim_generic() {
+ local dir keep k
+ find . -depth -type d \( -name test -o -name tests -o -name testdata \
+ -o -name doc -o -name docs -o -name example -o -name examples \
+ -o -name benchmark -o -name benchmarks -o -name bench -o -name fuzz \) \
+ -print | while read -r dir; do
+ keep=false
+ for k in "$@"; do
+ if [ "${dir}" = "./${k}" ]; then
+ keep=true
+ fi
+ done
+ if [ "${keep}" = "false" ]; then
+ rm -rf "${dir}"
+ fi
+ done
+}
+
+# Trim an extracted source tree (the current directory) for the dependency
+# named $1 (e.g. "thrift")
+trim_dependency() {
+ case "$1" in
+ brotli)
+ rmd go python java csharp js research
+ # install() copies the man pages from docs/
+ trim_generic docs
+ ;;
+ bzip2)
+ rm -f manual.* sample* ./*.html
+ trim_generic
+ ;;
+ lz4)
+ # Only lib/ is built (LZ4_BUILD_CLI=OFF). Files outside of lib/ are
+ # GPL-2.0-or-later, so don't ship the ones we don't need.
+ rmd programs contrib ossfuzz
+ trim_generic
+ (cd build && keep_only cmake)
+ ;;
+ mimalloc)
+ rmd ide
+ trim_generic
+ ;;
+ rapidjson)
+ rmd bin thirdparty
+ trim_generic
+ ;;
+ re2)
+ rmd benchlog python app
+ trim_generic
+ ;;
+ simdjson)
+ # Built from src/ and include/. singleheader/ is a 20 MB amalgamation.
+ rmd singleheader images jsonexamples tools scripts
+ trim_generic
+ ;;
+ thrift)
+ # Only the C++ library is built (BUILD_COMPILER=OFF, WITH_*=OFF)
+ rmd tutorial test compiler contrib doc debian
+ (cd lib && keep_only cpp CMakeLists.txt)
+ trim_generic
+ ;;
+ zlib)
+ rmd contrib
+ # zlib's CMakeLists.txt builds test/example.c unless ZLIB_BUILD_EXAMPLES=OFF
+ trim_generic test
+ ;;
+ zstd)
+ # ZSTD_BUILD_PROGRAMS=OFF
+ rmd contrib zlibWrapper programs
+ trim_generic
+ (cd build && keep_only cmake)
+ ;;
+ *)
+ trim_generic
+ ;;
+ esac
+}
+
+# Extract $1, trim it for dependency $3 and write the result to $2
+trim_tarball() {
+ local in="$1" out="$2" name="$3"
+ local work src
+ if [ -f "${SOURCE_DIR}/trim/${name}.sh" ]; then
+ bash "${SOURCE_DIR}/trim/${name}.sh" "${in}" "${out}"
+ return
+ fi
+ work="$(mktemp -d)"
+ # Use CMake's tar (libarchive) for .tar.bz2: tar -j needs a bzip2 binary,
+ # which e.g. the R CI images don't have.
+ case "${in}" in
+ *.tar.bz2) (cd "${work}" && cmake -E tar xf "${in}") ;;
+ *) tar -xf "${in}" -C "${work}" ;;
+ esac
+ # Every tarball we vendor has a single top-level directory
+ src="$(cd "${work}" && ls)"
+ if [ "$(echo "${src}" | wc -l)" -ne 1 ] || [ ! -d "${work}/${src}" ]; then
+ echo "Expected a single top-level directory in ${in}" >&2
+ rm -rf "${work}"
+ return 1
+ fi
+ (cd "${work}/${src}" && trim_dependency "${name}")
+ case "${out}" in
+ *.tar.bz2) (cd "${work}" && cmake -E tar cjf "${out}" "${src}") ;;
+ *.tar.gz) tar -czf "${out}" -C "${work}" "${src}" ;;
+ *)
+ echo "Don't know how to compress ${out}" >&2
+ rm -rf "${work}"
+ return 1
+ ;;
+ esac
+ rm -rf "${work}"
+}
+
+mkdir -p "${DESTDIR}" "${DOWNLOAD_DIR}"
+# trim_tarball() changes directories, so it needs absolute paths
+DOWNLOAD_DIR="$(cd "${DOWNLOAD_DIR}" && pwd)"
+TRIM_DIR="$(mktemp -d)"
+trap 'rm -rf "${TRIM_DIR}"' EXIT
+keep=(checksums.txt)
+sums="filename upstream_sha256 vendored_sha256"$'\n'
for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do
url=""
for dep_packed in "${DEPENDENCIES[@]}"; do
@@ -67,28 +259,38 @@ for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do
echo "${vendored_url_var} not found in versions.txt" >&2
exit 1
fi
- # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM
+ # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM and xsimd
checksum_var="${vendored_url_var%_URL}_BUILD_SHA256_CHECKSUM"
checksum="${!checksum_var}"
+ name="$(echo "${vendored_url_var}" | sed -e 's/^ARROW_//' -e 's/_URL$//' | tr '[:upper:]' '[:lower:]')"
keep+=("${tar_name}")
- out="${DESTDIR}/${tar_name}"
- if [ -f "${out}" ] && [ "$(sha256 "${out}")" = "${checksum}" ]; then
- continue
+ upstream="${DOWNLOAD_DIR}/${tar_name}"
+ if [ ! -f "${upstream}" ] || [ "$(sha256 "${upstream}")" != "${checksum}" ]; then
+ echo "Downloading ${url}"
+ curl --fail --silent --show-error --location --retry 3 --output "${upstream}" "${url}"
+ if [ "$(sha256 "${upstream}")" != "${checksum}" ]; then
+ echo "Checksum mismatch for ${upstream}" >&2
+ rm -f "${upstream}"
+ exit 1
+ fi
fi
- echo "Downloading ${url}"
- curl --fail --silent --show-error --location --retry 3 --output "${out}" "${url}"
-
- if [ "$(sha256 "${out}")" != "${checksum}" ]; then
- echo "Checksum mismatch for ${out}" >&2
- rm -f "${out}"
- exit 1
- fi
+ # Trim into a temporary directory (keeping the file name, so the extension
+ # says how to compress it) so that a failure doesn't leave a partial file.
+ echo "Trimming ${tar_name}"
+ trim_tarball "${upstream}" "${TRIM_DIR}/${tar_name}" "${name}"
+ out="${DESTDIR}/${tar_name}"
+ mv "${TRIM_DIR}/${tar_name}" "${out}"
+ sums="${sums}${tar_name} ${checksum} $(sha256 "${out}")"$'\n'
done
-# Remove tarballs for other versions
+# See read_vendored_checksums() in nixlibs.R
+printf "%s" "${sums}" > "${DESTDIR}/checksums.txt"
+
+# Remove tarballs for other versions (or dependencies we no longer vendor)
for file in "${DESTDIR}"/*; do
+ [ -e "${file}" ] || continue
stale=true
for name in "${keep[@]}"; do
if [ "$(basename "${file}")" = "${name}" ]; then