GH-55454: [R][C++] Vendor URI parser for the R library

ci/scripts/r_test.sh

@@ -91,6 +91,22 @@ fi
 export TEXMFCONFIG=/tmp/texmf-config
 export TEXMFVAR=/tmp/texmf-var
 
+if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then
+  # Simulate a machine without network access (e.g. a CRAN builder whose
+  # downloads fail) by disabling DNS resolution, like ubuntu-cpp-bundled-offline.
+  # We can't use mv because we can't remove /etc/resolv.conf in Docker
+  # container.
+  cp /etc/resolv.conf{,.bak}
+  echo > /etc/resolv.conf
+  # sccache can't reach its remote cache without network access
+  unset SCCACHE_BUCKET SCCACHE_REGION SCCACHE_S3_KEY_PREFIX
+  # Make sure we actually are offline so this check can't silently pass
+  if ${R_BIN} -s -e 'readLines(url("https://github.com"), n = 1L)' > /dev/null 2>&1; then
+    echo "Network is still reachable, but ARROW_R_TEST_OFFLINE=true"
+    exit 1
+  fi
+fi
+
 # Make sure we aren't writing to the home dir (CRAN _hates_ this but there is no official check)
 BEFORE=$(ls -alh ~/)
 
@@ -125,13 +141,23 @@ SCRIPT="as_cran <- !identical(tolower(Sys.getenv('NOT_CRAN')), 'true')
     args <- c(args, paste0('--install-args=\"', install_args, '\"'))
   }
 
+  # Without network access, some R CMD check steps (e.g. filtering undeclared
+  # '::' imports in tests against CRAN, resolving Rd xrefs) produce spurious
+  # warnings, so only fail on errors when offline.
+  offline <- identical(tolower(Sys.getenv('ARROW_R_TEST_OFFLINE')), 'true')
+  error_on <- if (offline) 'error' else 'warning'
+
   message('Running rcmdcheck with:\n')
   print(build_args)
   print(args)
 
-  rcmdcheck::rcmdcheck(build_args = build_args, args = args, error_on = 'warning', check_dir = 'check', timeout = 3600)"
+  rcmdcheck::rcmdcheck(build_args = build_args, args = args, error_on = error_on, check_dir = 'check', timeout = 3600)"
 echo "$SCRIPT" | "${R_BIN}" --no-save
 
+if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then
+  cp /etc/resolv.conf{.bak,}
+fi
+
 AFTER=$(ls -alh ~/)
 if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then
   # Ignore ~/.TinyTex/ and ~/R/ because it has many files.

dev/tasks/tasks.yml

@@ -632,6 +632,16 @@ tasks:
     ci: github
     template: r/github.linux.offline.build.yml
 
+  # Simulate a CRAN builder without network access: the build should detect
+  # that it is offline and fall back to a minimal build without downloading
+  # anything.
+  test-r-offline-minimal:
+    ci: github
+    template: docker-tests/github.linux.yml
+    params:
+      flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true'
+      image: r
+
   # This one has -flto=auto
   test-r-ubuntu-22.04:
     ci: github

r/.Rbuildignore

@@ -6,7 +6,7 @@ LICENSE.md
 ^data-raw$
 lint.sh
 Dockerfile
-.*\.tar\.gz
+^[^/]*\.tar\.gz$
 ^windows
 ^libarrow
 ^revdep

r/.gitignore

@@ -32,3 +32,6 @@ extra-tests/files
 /tools/LICENSE.txt
 /tools/NOTICE.txt
 /tools/versions.txt
+# Source tarballs of required dependencies that we ship for offline builds.
+# They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh).
+/tools/vendored/

r/Makefile

@@ -47,6 +47,7 @@ sync-cpp:
 	cp -p ../NOTICE.txt tools/
 	cp -p ../LICENSE.txt tools/
 	sed -i"" -e "s/\.env/dotenv/g" tools/cpp/CMakeLists.txt
+	bash tools/vendor_dependencies.sh
 
 build:
 	$(MAKE) clean

r/tools/nixlibs.R

@@ -646,6 +646,9 @@ build_libarrow <- function(src_dir, dst_dir) {
     # Add the *_SOURCE_URL env vars
     env_var_list <- set_thirdparty_urls(env_var_list)
   }
+  # Some thirdparty dependencies are required even for a minimal build, so they
+  # can't be turned off like the ones above.
+  env_var_list <- with_vendored_dependencies(env_var_list)
   env_vars <- env_vars_as_string(env_var_list)
 
   lg("arrow %s", ifelse(quietly, "", paste("with", env_vars)), .indent = "****")
@@ -871,6 +874,39 @@ set_thirdparty_urls <- function(env_var_list) {
   env_var_list
 }
 
+# libarrow requires some thirdparty dependencies, even for a minimal build.
+# The libarrow build uses them from the system if they are available and
+# otherwise builds them from source, downloading from ARROW_*_URL. To make sure
+# that works without network access, `make sync-cpp` saves their source
+# tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here.
+# This does not change whether the system versions are used.
+with_vendored_dependencies <- function(env_var_list) {
+  if (!dir.exists(vendored_dependency_dir)) {
+    # e.g. building from a git checkout without `make sync-cpp`
+    if (!quietly) {
+      lg("No vendored dependencies found, they will be built with CMake if needed", .indent = "****")
+    }
+    return(env_var_list)
+  }
+  deps_df <- get_component_names()
+  for (env_varname in vendored_dependencies) {
+    if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) {
+      # Respect the user's (or set_thirdparty_urls()'s) choice
+      next
+    }
+    filename <- deps_df$filename[deps_df$env_varname == env_varname]
+    tarball <- file.path(vendored_dependency_dir, filename)
+    if (length(tarball) != 1 || !file.exists(tarball)) {
+      if (!quietly) {
+        lg("Vendored source for %s not found, it will be downloaded if needed", env_varname, .indent = "****")
+      }
+      next
+    }
+    env_var_list <- replace(env_var_list, env_varname, normalizePath(tarball))
+  }
+  env_var_list
+}
+
 # this is generally about features that people asked for via environment variables, but
 # for some cases (like S3 when we override it in this script) we might find those in
 # env_var_list
@@ -1104,6 +1140,12 @@ if (download_libarrow_ok && !is_release && !test_mode) {
 # `create_package_with_all_dependencies()` in install-arrow.R
 thirdparty_dependency_dir <- Sys.getenv("ARROW_THIRDPARTY_DEPENDENCY_DIR", "tools/thirdparty_dependencies")
 
+# Source tarballs of dependencies that libarrow requires even for a minimal build.
+# These are always shipped in the package (see `with_vendored_dependencies()`).
+# If you change this path or list, you also need to edit tools/vendor_dependencies.sh
+vendored_dependency_dir <- "tools/vendored"
+vendored_dependencies <- c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL")
+
 arrow_versioned <- paste0("arrow-", VERSION)
 # configure.win uses a different libarrow dir and the zip is already nested
 if (on_windows) {

r/tools/test-nixlibs.R

@@ -232,3 +232,59 @@ test_that("find_latest_nightly()", {
     "Failed to find latest nightly"
   )
 })
+
+test_that("with_vendored_dependencies()", {
+  vendored_dir <- tempfile()
+  dir.create(vendored_dir)
+  on.exit(unlink(vendored_dir, recursive = TRUE), add = TRUE)
+  original_dir <- nixlibs_env$vendored_dependency_dir
+  original_get_component_names <- nixlibs_env$get_component_names
+  original_quietly <- nixlibs_env$quietly
+  on.exit(nixlibs_env$vendored_dependency_dir <- original_dir, add = TRUE)
+  on.exit(nixlibs_env$get_component_names <- original_get_component_names, add = TRUE)
+  on.exit(nixlibs_env$quietly <- original_quietly, add = TRUE)
+  nixlibs_env$vendored_dependency_dir <- vendored_dir
+  nixlibs_env$get_component_names <- function() {
+    data.frame(
+      env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL"),
+      filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz")
+    )
+  }
+  withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA))
+
+  # Without the vendored tarballs, nothing is set
+  nixlibs_env$quietly <- TRUE
+  expect_silent(
+    expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1"))
+  )
+  # The missing tarballs are only reported with verbose logging
+  nixlibs_env$quietly <- FALSE
+  expect_output(
+    expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")),
+    "Vendored source for ARROW_URIPARSER_URL not found"
+  )
+  nixlibs_env$quietly <- TRUE
+
+  uriparser <- file.path(vendored_dir, "uriparser-1.0.2.tar.bz2")
+  xsimd <- file.path(vendored_dir, "xsimd-14.3.0.tar.gz")
+  # Only dependencies in vendored_dependencies are used
+  zstd <- file.path(vendored_dir, "zstd-1.5.7.tar.gz")
+  file.create(uriparser, xsimd, zstd)
+  expect_identical(
+    with_vendored_dependencies(list(A = "1")),
+    list(A = "1", ARROW_URIPARSER_URL = normalizePath(uriparser), ARROW_XSIMD_URL = normalizePath(xsimd))
+  )
+
+  # Already set (e.g. by set_thirdparty_urls()) is respected
+  expect_identical(
+    with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2")),
+    list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = normalizePath(xsimd))
+  )
+
+  # As is a user-set env var
+  withr::local_envvar(c(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2"))
+  expect_identical(
+    with_vendored_dependencies(list(A = "1")),
+    list(A = "1", ARROW_XSIMD_URL = normalizePath(xsimd))
+  )
+})

r/tools/vendor_dependencies.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# libarrow requires some thirdparty dependencies, even for a minimal build.
+# This script downloads the source tarballs that libarrow would otherwise
+# download at build time, so that they can be shipped in the R package.
+# nixlibs.R uses them when the dependencies are not found on the system, so
+# that building without network access (e.g. on CRAN) still works.
+#
+# Usage: tools/vendor_dependencies.sh [DESTDIR]
+# Must be run after tools/cpp has been synced (see `make sync-cpp`).
+
+set -euo pipefail
+
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DESTDIR="${1:-${SOURCE_DIR}/vendored}"
+
+# If you change this list, you also need to edit
+# `vendored_dependencies` in nixlibs.R
+VENDORED_DEPENDENCIES=(
+  ARROW_URIPARSER_URL
+  ARROW_XSIMD_URL
+)
+
+# Load `DEPENDENCIES` and the *_BUILD_SHA256_CHECKSUM variables.
+# shellcheck source=/dev/null
+source "${SOURCE_DIR}/cpp/thirdparty/versions.txt"
+
+sha256() {
+  if command -v sha256sum > /dev/null 2>&1; then
+    sha256sum "$1" | cut -d " " -f 1
+  else
+    shasum -a 256 "$1" | cut -d " " -f 1
+  fi
+}
+
+mkdir -p "${DESTDIR}"
+keep=()
+for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do
+  url=""
+  for dep_packed in "${DEPENDENCIES[@]}"; do
+    # Unpack each entry of the form "$home_var $tar_out $dep_url"
+    IFS=" " read -r dep_url_var dep_tar_name dep_url <<< "${dep_packed}"
+    if [ "${dep_url_var}" = "${vendored_url_var}" ]; then
+      tar_name="${dep_tar_name}"
+      url="${dep_url}"
+      break
+    fi
+  done
+  if [ -z "${url}" ]; then
+    echo "${vendored_url_var} not found in versions.txt" >&2
+    exit 1
+  fi
+  # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM
+  checksum_var="${vendored_url_var%_URL}_BUILD_SHA256_CHECKSUM"
+  checksum="${!checksum_var}"
+  keep+=("${tar_name}")
+
+  out="${DESTDIR}/${tar_name}"
+  if [ -f "${out}" ] && [ "$(sha256 "${out}")" = "${checksum}" ]; then
+    continue
+  fi
+
+  echo "Downloading ${url}"
+  curl --fail --silent --show-error --location --retry 3 --output "${out}" "${url}"
+
+  if [ "$(sha256 "${out}")" != "${checksum}" ]; then
+    echo "Checksum mismatch for ${out}" >&2
+    rm -f "${out}"
+    exit 1
+  fi
+done
+
+# Remove tarballs for other versions
+for file in "${DESTDIR}"/*; do
+  stale=true
+  for name in "${keep[@]}"; do
+    if [ "$(basename "${file}")" = "${name}" ]; then
+      stale=false
+    fi
+  done
+  if [ "${stale}" = "true" ]; then
+    rm -f "${file}"
+  fi
+done