From 7b051a665cbb8cee32b90df1dc4ca9d56c9a064b Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 9 Nov 2026 17:16:20 -0600 Subject: [PATCH 1/6] A test --- ci/scripts/r_test.sh | 20 ++++++++++++++++++++ dev/tasks/tasks.yml | 10 ++++++++++ 2 files changed, 30 insertions(+) diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 80e7d5dc4296..f4114a14d930 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -91,6 +91,22 @@ fi export TEXMFCONFIG=/tmp/texmf-config export TEXMFVAR=/tmp/texmf-var +if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then + # Simulate a machine without network access (e.g. a CRAN builder whose + # downloads fail) by disabling DNS resolution, like ubuntu-cpp-bundled-offline. + # We can't use mv because we can't remove /etc/resolv.conf in Docker + # container. + cp /etc/resolv.conf{,.bak} + echo > /etc/resolv.conf + # sccache can't reach its remote cache without network access + unset SCCACHE_BUCKET SCCACHE_REGION SCCACHE_S3_KEY_PREFIX + # Make sure we actually are offline so this check can't silently pass + if ${R_BIN} -s -e 'readLines(url("https://github.com"), n = 1L)' > /dev/null 2>&1; then + echo "Network is still reachable, but ARROW_R_TEST_OFFLINE=true" + exit 1 + fi +fi + # Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check) BEFORE=$(ls -alh ~/) @@ -132,6 +148,10 @@ SCRIPT="as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true') rcmdcheck::rcmdcheck(build_args = build_args, args = args, error_on = 'warning', check_dir = 'check', timeout = 3600)" echo "$SCRIPT" | "${R_BIN}" --no-save +if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then + cp /etc/resolv.conf{.bak,} +fi + AFTER=$(ls -alh ~/) if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then # Ignore ~/.TinyTex/ and ~/R/ because it has many files. diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 0e7d7bd296b9..d9351d3afe0a 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -632,6 +632,16 @@ tasks: ci: github template: r/github.linux.offline.build.yml + # Simulate a CRAN builder without network access: the build should detect + # that it is offline and fall back to a minimal build without downloading + # anything. + test-r-offline-minimal: + ci: github + template: docker-tests/github.linux.yml + params: + flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true' + image: r + # This one has -flto=auto test-r-ubuntu-22.04: ci: github From cd7d74e720bc4878fd5ee1ac637751fa3faf3346 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 9 Nov 2026 19:21:31 -0600 Subject: [PATCH 2/6] vendor xsimd+uriparse for the r package --- r/.Rbuildignore | 2 +- r/.gitignore | 3 + r/Makefile | 1 + r/tools/nixlibs.R | 33 +++++++++++ r/tools/test-nixlibs.R | 47 +++++++++++++++ r/tools/vendor_dependencies.sh | 101 +++++++++++++++++++++++++++++++++ 6 files changed, 186 insertions(+), 1 deletion(-) create mode 100644 r/tools/vendor_dependencies.sh diff --git a/r/.Rbuildignore b/r/.Rbuildignore index 87861e04e686..deaf5dd3948c 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -6,7 +6,7 @@ LICENSE.md ^data-raw$ lint.sh Dockerfile -.*\.tar\.gz +^[^/]*\.tar\.gz$ ^windows ^libarrow ^revdep diff --git a/r/.gitignore b/r/.gitignore index d680fb42ddd6..a1295ea4e0b5 100644 --- a/r/.gitignore +++ b/r/.gitignore @@ -32,3 +32,6 @@ extra-tests/files /tools/LICENSE.txt /tools/NOTICE.txt /tools/versions.txt +# Source tarballs of required dependencies that we ship for offline builds. +# They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh). +/tools/vendored/ diff --git a/r/Makefile b/r/Makefile index 12d61b70c920..27d4683a1800 100644 --- a/r/Makefile +++ b/r/Makefile @@ -47,6 +47,7 @@ sync-cpp: cp -p ../NOTICE.txt tools/ cp -p ../LICENSE.txt tools/ sed -i"" -e "s/\.env/dotenv/g" tools/cpp/CMakeLists.txt + bash tools/vendor_dependencies.sh build: $(MAKE) clean diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 3c184f6fe232..233aa4c67a75 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -646,6 +646,9 @@ build_libarrow <- function(src_dir, dst_dir) { # Add the *_SOURCE_URL env vars env_var_list <- set_thirdparty_urls(env_var_list) } + # Some thirdparty dependencies are required even for a minimal build, so they + # can't be turned off like the ones above. + env_var_list <- with_vendored_dependencies(env_var_list) env_vars <- env_vars_as_string(env_var_list) lg("arrow %s", ifelse(quietly, "", paste("with", env_vars)), .indent = "****") @@ -871,6 +874,30 @@ set_thirdparty_urls <- function(env_var_list) { env_var_list } +with_vendored_dependencies <- function(env_var_list) { + # libarrow requires some thirdparty dependencies, even for a minimal build. + # The libarrow build uses them from the system if they are available and + # otherwise builds them from source, downloading from ARROW_*_URL. To make sure + # that works without network access, `make sync-cpp` saves their source + # tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here. + # This does not change whether the system versions are used. + deps_df <- get_component_names() + for (env_varname in vendored_dependencies) { + if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { + # Respect the user's (or set_thirdparty_urls()'s) choice + next + } + filename <- deps_df$filename[deps_df$env_varname == env_varname] + tarball <- file.path(vendored_dependency_dir, filename) + if (length(tarball) != 1 || !file.exists(tarball)) { + lg("Vendored source for %s not found, it will be downloaded if needed", env_varname, .indent = "****") + next + } + env_var_list <- replace(env_var_list, env_varname, normalizePath(tarball)) + } + env_var_list +} + # this is generally about features that people asked for via environment variables, but # for some cases (like S3 when we override it in this script) we might find those in # env_var_list @@ -1104,6 +1131,12 @@ if (download_libarrow_ok && !is_release && !test_mode) { # `create_package_with_all_dependencies()` in install-arrow.R thirdparty_dependency_dir <- Sys.getenv("ARROW_THIRDPARTY_DEPENDENCY_DIR", "tools/thirdparty_dependencies") +# Source tarballs of dependencies that libarrow requires even for a minimal build. +# These are always shipped in the package (see `with_vendored_dependencies()`). +# If you change this path or list, you also need to edit tools/vendor_dependencies.sh +vendored_dependency_dir <- "tools/vendored" +vendored_dependencies <- c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL") + arrow_versioned <- paste0("arrow-", VERSION) # configure.win uses a different libarrow dir and the zip is already nested if (on_windows) { diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index f7711c97ce47..87792f16f7d3 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -232,3 +232,50 @@ test_that("find_latest_nightly()", { "Failed to find latest nightly" ) }) + +test_that("with_vendored_dependencies()", { + vendored_dir <- tempfile() + dir.create(vendored_dir) + on.exit(unlink(vendored_dir, recursive = TRUE), add = TRUE) + original_dir <- nixlibs_env$vendored_dependency_dir + original_get_component_names <- nixlibs_env$get_component_names + on.exit(nixlibs_env$vendored_dependency_dir <- original_dir, add = TRUE) + on.exit(nixlibs_env$get_component_names <- original_get_component_names, add = TRUE) + nixlibs_env$vendored_dependency_dir <- vendored_dir + nixlibs_env$get_component_names <- function() { + data.frame( + env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL"), + filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz") + ) + } + withr::local_envvar(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA) + + # Without the vendored tarballs, nothing is set + expect_output( + expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")), + "Vendored source for ARROW_URIPARSER_URL not found" + ) + + uriparser <- file.path(vendored_dir, "uriparser-1.0.2.tar.bz2") + xsimd <- file.path(vendored_dir, "xsimd-14.3.0.tar.gz") + # Only dependencies in vendored_dependencies are used + zstd <- file.path(vendored_dir, "zstd-1.5.7.tar.gz") + file.create(uriparser, xsimd, zstd) + expect_identical( + with_vendored_dependencies(list(A = "1")), + list(A = "1", ARROW_URIPARSER_URL = normalizePath(uriparser), ARROW_XSIMD_URL = normalizePath(xsimd)) + ) + + # Already set (e.g. by set_thirdparty_urls()) is respected + expect_identical( + with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2")), + list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = normalizePath(xsimd)) + ) + + # As is a user-set env var + withr::local_envvar(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2") + expect_identical( + with_vendored_dependencies(list(A = "1")), + list(A = "1", ARROW_XSIMD_URL = normalizePath(xsimd)) + ) +}) diff --git a/r/tools/vendor_dependencies.sh b/r/tools/vendor_dependencies.sh new file mode 100644 index 000000000000..caa28bff5ce2 --- /dev/null +++ b/r/tools/vendor_dependencies.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# libarrow requires some thirdparty dependencies, even for a minimal build. +# This script downloads the source tarballs that libarrow would otherwise +# download at build time, so that they can be shipped in the R package. +# nixlibs.R uses them when the dependencies are not found on the system, so +# that building without network access (e.g. on CRAN) still works. +# +# Usage: tools/vendor_dependencies.sh [DESTDIR] +# Must be run after tools/cpp has been synced (see `make sync-cpp`). + +set -euo pipefail + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DESTDIR="${1:-${SOURCE_DIR}/vendored}" + +# If you change this list, you also need to edit +# `vendored_dependencies` in nixlibs.R +VENDORED_DEPENDENCIES=( + ARROW_URIPARSER_URL + ARROW_XSIMD_URL +) + +# Load `DEPENDENCIES` and the *_BUILD_SHA256_CHECKSUM variables. +# shellcheck source=/dev/null +source "${SOURCE_DIR}/cpp/thirdparty/versions.txt" + +sha256() { + if command -v sha256sum > /dev/null 2>&1; then + sha256sum "$1" | cut -d " " -f 1 + else + shasum -a 256 "$1" | cut -d " " -f 1 + fi +} + +mkdir -p "${DESTDIR}" +keep=() +for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do + url="" + for dep_packed in "${DEPENDENCIES[@]}"; do + # Unpack each entry of the form "$home_var $tar_out $dep_url" + IFS=" " read -r dep_url_var dep_tar_name dep_url <<< "${dep_packed}" + if [ "${dep_url_var}" = "${vendored_url_var}" ]; then + tar_name="${dep_tar_name}" + url="${dep_url}" + break + fi + done + if [ -z "${url}" ]; then + echo "${vendored_url_var} not found in versions.txt" >&2 + exit 1 + fi + # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM + checksum_var="${vendored_url_var%_URL}_BUILD_SHA256_CHECKSUM" + checksum="${!checksum_var}" + keep+=("${tar_name}") + + out="${DESTDIR}/${tar_name}" + if [ -f "${out}" ] && [ "$(sha256 "${out}")" = "${checksum}" ]; then + continue + fi + + echo "Downloading ${url}" + curl --fail --silent --show-error --location --retry 3 --output "${out}" "${url}" + + if [ "$(sha256 "${out}")" != "${checksum}" ]; then + echo "Checksum mismatch for ${out}" >&2 + rm -f "${out}" + exit 1 + fi +done + +# Remove tarballs for other versions +for file in "${DESTDIR}"/*; do + stale=true + for name in "${keep[@]}"; do + if [ "$(basename "${file}")" = "${name}" ]; then + stale=false + fi + done + if [ "${stale}" = "true" ]; then + rm -f "${file}" + fi +done From 3fa979f8548292e26b90f691fc26d6bc8e45a33d Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Mon, 9 Nov 2026 19:59:26 -0600 Subject: [PATCH 3/6] copilot comments --- r/tools/nixlibs.R | 11 ++++++++++- r/tools/test-nixlibs.R | 13 +++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 233aa4c67a75..3db0740c35ec 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -881,6 +881,13 @@ with_vendored_dependencies <- function(env_var_list) { # that works without network access, `make sync-cpp` saves their source # tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here. # This does not change whether the system versions are used. + if (!dir.exists(vendored_dependency_dir)) { + # e.g. building from a git checkout without `make sync-cpp` + if (!quietly) { + lg("No vendored dependencies found, they will be downloaded if needed", .indent = "****") + } + return(env_var_list) + } deps_df <- get_component_names() for (env_varname in vendored_dependencies) { if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { @@ -890,7 +897,9 @@ with_vendored_dependencies <- function(env_var_list) { filename <- deps_df$filename[deps_df$env_varname == env_varname] tarball <- file.path(vendored_dependency_dir, filename) if (length(tarball) != 1 || !file.exists(tarball)) { - lg("Vendored source for %s not found, it will be downloaded if needed", env_varname, .indent = "****") + if (!quietly) { + lg("Vendored source for %s not found, it will be downloaded if needed", env_varname, .indent = "****") + } next } env_var_list <- replace(env_var_list, env_varname, normalizePath(tarball)) diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index 87792f16f7d3..9d8abccfbb5e 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -239,8 +239,10 @@ test_that("with_vendored_dependencies()", { on.exit(unlink(vendored_dir, recursive = TRUE), add = TRUE) original_dir <- nixlibs_env$vendored_dependency_dir original_get_component_names <- nixlibs_env$get_component_names + original_quietly <- nixlibs_env$quietly on.exit(nixlibs_env$vendored_dependency_dir <- original_dir, add = TRUE) on.exit(nixlibs_env$get_component_names <- original_get_component_names, add = TRUE) + on.exit(nixlibs_env$quietly <- original_quietly, add = TRUE) nixlibs_env$vendored_dependency_dir <- vendored_dir nixlibs_env$get_component_names <- function() { data.frame( @@ -248,13 +250,20 @@ test_that("with_vendored_dependencies()", { filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz") ) } - withr::local_envvar(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA) + withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) # Without the vendored tarballs, nothing is set + nixlibs_env$quietly <- TRUE + expect_silent( + expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")) + ) + # The missing tarballs are only reported with verbose logging + nixlibs_env$quietly <- FALSE expect_output( expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")), "Vendored source for ARROW_URIPARSER_URL not found" ) + nixlibs_env$quietly <- TRUE uriparser <- file.path(vendored_dir, "uriparser-1.0.2.tar.bz2") xsimd <- file.path(vendored_dir, "xsimd-14.3.0.tar.gz") @@ -273,7 +282,7 @@ test_that("with_vendored_dependencies()", { ) # As is a user-set env var - withr::local_envvar(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2") + withr::local_envvar(c(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2")) expect_identical( with_vendored_dependencies(list(A = "1")), list(A = "1", ARROW_XSIMD_URL = normalizePath(xsimd)) From ad9f37f0e6f18f9fa10176da7926110f7cb1b978 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 07:37:05 -0600 Subject: [PATCH 4/6] error on error offline --- ci/scripts/r_test.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index f4114a14d930..20e6062e847f 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -141,11 +141,17 @@ SCRIPT="as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true') args <- c(args, paste0('--install-args=\"', install_args, '\"')) } + # Without network access, some R CMD check steps (e.g. filtering undeclared + # '::' imports in tests against CRAN, resolving Rd xrefs) produce spurious + # warnings, so only fail on errors when offline. + offline <- identical(tolower(Sys.getenv('ARROW_R_TEST_OFFLINE')), 'true') + error_on <- if (offline) 'error' else 'warning' + message('Running rcmdcheck with:\n') print(build_args) print(args) - rcmdcheck::rcmdcheck(build_args = build_args, args = args, error_on = 'warning', check_dir = 'check', timeout = 3600)" + rcmdcheck::rcmdcheck(build_args = build_args, args = args, error_on = error_on, check_dir = 'check', timeout = 3600)" echo "$SCRIPT" | "${R_BIN}" --no-save if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then From e758fef55bb8f9e11c32ea95520b1c01f6496031 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 07:42:35 -0600 Subject: [PATCH 5/6] Apply suggestion from @jonkeane --- r/tools/nixlibs.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 3db0740c35ec..98c97364f48e 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -874,13 +874,13 @@ set_thirdparty_urls <- function(env_var_list) { env_var_list } +# libarrow requires some thirdparty dependencies, even for a minimal build. +# The libarrow build uses them from the system if they are available and +# otherwise builds them from source, downloading from ARROW_*_URL. To make sure +# that works without network access, `make sync-cpp` saves their source +# tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here. +# This does not change whether the system versions are used. with_vendored_dependencies <- function(env_var_list) { - # libarrow requires some thirdparty dependencies, even for a minimal build. - # The libarrow build uses them from the system if they are available and - # otherwise builds them from source, downloading from ARROW_*_URL. To make sure - # that works without network access, `make sync-cpp` saves their source - # tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here. - # This does not change whether the system versions are used. if (!dir.exists(vendored_dependency_dir)) { # e.g. building from a git checkout without `make sync-cpp` if (!quietly) { From 212778c191cd466c87dc40ec8c2ba5d5992d0244 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Tue, 10 Nov 2026 07:57:41 -0600 Subject: [PATCH 6/6] Make the log line clearer when the vendored dependencies aren't there. --- r/tools/nixlibs.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 98c97364f48e..226894372ec5 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -884,7 +884,7 @@ with_vendored_dependencies <- function(env_var_list) { if (!dir.exists(vendored_dependency_dir)) { # e.g. building from a git checkout without `make sync-cpp` if (!quietly) { - lg("No vendored dependencies found, they will be downloaded if needed", .indent = "****") + lg("No vendored dependencies found, they will be built with CMake if needed", .indent = "****") } return(env_var_list) }