GH-55478: [R] Vendored dependencies to enable parquet (smallest change, biggest tarball)

ci/scripts/r_test.sh

@@ -158,6 +158,26 @@ if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then
   cp /etc/resolv.conf{.bak,}
 fi
 
+if [ -n "${ARROW_R_EXPECTED_FEATURES:-}" ]; then
+  # The tests skip what the build doesn't support, so make sure the features
+  # we expect (e.g. from the vendored thirdparty sources) were actually built.
+  # ARROW_R_EXPECTED_FEATURES is a comma-separated list of
+  # names(arrow::arrow_info()$capabilities) and memory pool backends
+  # (e.g. mimalloc).
+  ${R_BIN} -s -e "
+    library(arrow, lib.loc = 'check/arrow.Rcheck')
+    info <- arrow_info()
+    print(info\$capabilities)
+    print(info\$memory_pool\$available_backends)
+    available <- c(names(which(info\$capabilities)), info\$memory_pool\$available_backends)
+    expected <- strsplit(Sys.getenv('ARROW_R_EXPECTED_FEATURES'), ',')[[1]]
+    missing <- setdiff(expected, available)
+    if (length(missing) > 0) {
+      stop('Expected features are not available: ', paste(missing, collapse = ', '))
+    }
+  "
+fi
+
 AFTER=$(ls -alh ~/)
 if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then
   # Ignore ~/.TinyTex/ and ~/R/ because it has many files.

dev/release/rat_exclude_files.txt

@@ -97,6 +97,7 @@ r/tests/testthat/_snaps/*.md
 r/inst/include/cpp11.hpp
 r/inst/include/cpp11/*.hpp
 r/tools/nixlibs-allowlist.txt
+r/inst/COPYRIGHTS
 .gitattributes
 ruby/red-arrow/.yardopts
 ruby/red-arrow-format/lib/arrow-format/org/*

dev/tasks/tasks.yml

@@ -633,14 +633,30 @@ tasks:
     template: r/github.linux.offline.build.yml
 
   # Simulate a CRAN builder without network access: the build should detect
-  # that it is offline and fall back to a minimal build without downloading
-  # anything.
-  test-r-offline-minimal:
-    ci: github
-    template: docker-tests/github.linux.yml
-    params:
-      flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true'
+  # that it is offline and build libarrow without downloading anything, using
+  # the thirdparty sources vendored in the package (tools/vendored) to still
+  # include Parquet, Dataset, JSON, the compression codecs, etc.
+  # ARROW_SOURCE_HOME='' makes it use the C++ source copied into tools/cpp
+  # (like the CRAN package does), whose versions.txt has the checksums of the
+  # trimmed tarballs. The codecs that are off by default on Linux are turned on
+  # so that every vendored tarball is built.
+  test-r-offline-vendored:
+    ci: github
+    template: docker-tests/github.linux.yml
+    params:
+      flags: >-
+        -e ARROW_R_TEST_OFFLINE=true
+        -e NOT_CRAN=false
+        -e SKIP_VIGNETTES=true
+        -e ARROW_SOURCE_HOME=''
+        -e ARROW_WITH_BROTLI=ON
+        -e ARROW_WITH_BZ2=ON
+        -e ARROW_WITH_ZSTD=ON
+        -e ARROW_R_EXPECTED_FEATURES=dataset,parquet,json,mimalloc,re2,utf8proc,snappy,gzip,brotli,zstd,lz4,bz2
       image: r
+      # A full libarrow build with 2 cores (NOT_CRAN=false) and no sccache
+      # (it's offline)
+      timeout: 180
 
   # This one has -flto=auto
   test-r-ubuntu-22.04:

r/.Rbuildignore

@@ -32,3 +32,5 @@ STYLE.md
 ^inst/__pycache__$
 ^bootstrap.R$
 air.toml
+^tools/vendored-downloads$
+^tools/trim$

r/.gitignore

@@ -35,3 +35,5 @@ extra-tests/files
 # Source tarballs of required dependencies that we ship for offline builds.
 # They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh).
 /tools/vendored/
+# Cache of the upstream (untrimmed) tarballs used by tools/vendor_dependencies.sh
+/tools/vendored-downloads/

r/DESCRIPTION

@@ -24,6 +24,7 @@ Description: 'Apache' 'Arrow' <https://arrow.apache.org/> is a cross-language
     package provides an interface to the 'Arrow C++' library.
 Depends: R (>= 4.2)
 License: Apache License (>= 2.0)
+Copyright: See file inst/COPYRIGHTS
 URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/
 BugReports: https://github.com/apache/arrow/issues
 Encoding: UTF-8

r/inst/COPYRIGHTS

@@ -0,0 +1,78 @@
+The arrow R package is Copyright The Apache Software Foundation and is licensed
+under the Apache License, Version 2.0. The Arrow C++ library source copied into
+tools/cpp includes code from other projects; see tools/LICENSE.txt and
+tools/NOTICE.txt in the source package.
+
+To build the Arrow C++ library without network access, the source package also
+includes (trimmed) source tarballs of these thirdparty dependencies in
+tools/vendored. They are only used when the dependency isn't found on the
+system, and are statically linked into the package when used. Each tarball
+contains the dependency's full license text.
+
+Boost (headers used by Apache Thrift)
+  Copyright the Boost authors
+  Boost Software License 1.0
+
+Brotli
+  Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors
+  MIT License
+
+bzip2
+  Copyright (C) 1996-2019 Julian R Seward
+  BSD-style license (see LICENSE in the tarball)
+
+LZ4 (lib/ only)
+  Copyright (c) 2011-2020, Yann Collet
+  BSD 2-Clause License
+
+mimalloc
+  Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen
+  MIT License
+
+RapidJSON
+  Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip
+  MIT License (with code under the BSD and JSON licenses, see license.txt in
+  the tarball)
+
+RE2
+  Copyright (c) 2009 The RE2 Authors
+  BSD 3-Clause License
+
+simdjson
+  Copyright 2018-2025 The simdjson authors
+  Apache License 2.0 or MIT License
+
+Snappy
+  Copyright 2011, Google Inc.
+  BSD 3-Clause License
+
+Apache Thrift
+  Copyright (C) 2006-2019, The Apache Software Foundation
+  Apache License 2.0
+
+uriparser (library sources only)
+  Copyright (C) 2007, Weijia Song and Sebastian Pipping
+  BSD 3-Clause License
+
+utf8proc
+  Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Tony Kelman,
+  Jonas Fonseca, and other contributors
+  Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany
+  MIT License, with Unicode data under the Unicode Data Files and Software
+  License
+
+xsimd
+  Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and
+  Martin Renou
+  Copyright (c) 2016, QuantStack
+  Copyright (c) 2018, Serge Guelton
+  BSD 3-Clause License
+
+zlib
+  Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler
+  zlib License
+
+Zstandard (zstd)
+  Copyright (c) Meta Platforms, Inc. and affiliates
+  BSD 3-Clause License (zstd is dual-licensed BSD/GPLv2; it is used under the
+  BSD license)