diff --git a/ci/scripts/r_test.sh b/ci/scripts/r_test.sh index 20e6062e847f..ac9fbf2c479d 100755 --- a/ci/scripts/r_test.sh +++ b/ci/scripts/r_test.sh @@ -158,6 +158,26 @@ if [ "${ARROW_R_TEST_OFFLINE:-false}" = "true" ]; then cp /etc/resolv.conf{.bak,} fi +if [ -n "${ARROW_R_EXPECTED_FEATURES:-}" ]; then + # The tests skip what the build doesn't support, so make sure the features + # we expect (e.g. from the vendored thirdparty sources) were actually built. + # ARROW_R_EXPECTED_FEATURES is a comma-separated list of + # names(arrow::arrow_info()$capabilities) and memory pool backends + # (e.g. mimalloc). + ${R_BIN} -s -e " + library(arrow, lib.loc = 'check/arrow.Rcheck') + info <- arrow_info() + print(info\$capabilities) + print(info\$memory_pool\$available_backends) + available <- c(names(which(info\$capabilities)), info\$memory_pool\$available_backends) + expected <- strsplit(Sys.getenv('ARROW_R_EXPECTED_FEATURES'), ',')[[1]] + missing <- setdiff(expected, available) + if (length(missing) > 0) { + stop('Expected features are not available: ', paste(missing, collapse = ', ')) + } + " +fi + AFTER=$(ls -alh ~/) if [ "$NOT_CRAN" != "true" ] && [ "$BEFORE" != "$AFTER" ]; then # Ignore ~/.TinyTex/ and ~/R/ because it has many files. diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 123def5b66c5..d067675b41ed 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -1085,7 +1085,63 @@ endif() # ---------------------------------------------------------------------- # Add Boost dependencies (code adapted from Apache Kudu) +# Use the bundled Boost source archive header-only, without Boost's +# CMake build. This only provides the Boost libraries that Apache +# Thrift needs. See ARROW_BOOST_FOR_THRIFT_ONLY. +# +# This is a macro to set variables in build_boost()'s PARENT_SCOPE. +macro(build_boost_headers_for_thrift) + list(APPEND CMAKE_MESSAGE_INDENT "Boost: ") + message(STATUS "Using headers from source (only for Apache Thrift)") + + fetchcontent_declare(boost + ${FC_DECLARE_COMMON_OPTIONS} + URL ${BOOST_SOURCE_URL} + URL_HASH "SHA256=${ARROW_BOOST_BUILD_SHA256_CHECKSUM}" + # This doesn't exist. So FetchContent only + # extracts Boost and doesn't add Boost's CMake + # project. + SOURCE_SUBDIR "arrow-header-only") + prepare_fetchcontent() + fetchcontent_makeavailable(boost) + + # Boost.Uuid (used by thrift/TUuid.h) and the Boost libraries it + # includes headers from. If you change this, check + # r/tools/trim/boost.sh too. + set(libraries + assert + config + throw_exception + type_traits + uuid) + set(boost_include_dirs) + foreach(library ${libraries}) + set(include_dir "${boost_SOURCE_DIR}/libs/${library}/include") + if(NOT IS_DIRECTORY "${include_dir}") + message(FATAL_ERROR "Boost source archive doesn't have ${include_dir}") + endif() + list(APPEND boost_include_dirs "${include_dir}") + endforeach() + add_library(Boost::headers INTERFACE IMPORTED) + target_include_directories(Boost::headers INTERFACE ${boost_include_dirs}) + target_compile_definitions(Boost::headers INTERFACE BOOST_ALL_NO_LIB) + + set(Boost_INCLUDE_DIRS + ${boost_include_dirs} + PARENT_SCOPE) + set(BOOST_VENDORED + TRUE + PARENT_SCOPE) + + list(POP_BACK CMAKE_MESSAGE_INDENT) +endmacro() + function(build_boost) + if(ARROW_BOOST_FOR_THRIFT_ONLY) + build_boost_headers_for_thrift() + return() + endif() + list(APPEND CMAKE_MESSAGE_INDENT "Boost: ") message(STATUS "Building from source") @@ -1321,6 +1377,30 @@ else() set(ARROW_USE_BOOST FALSE) endif() +# If Boost is only needed because Apache Thrift's headers use it, we +# can avoid both Boost's and Apache Thrift's CMake builds: bundled +# Boost is used header-only (see build_boost_headers_for_thrift()) and +# bundled Apache Thrift is built from only the sources that Apache +# Parquet needs (see build_thrift()). This is faster and lets packagers +# (e.g. the R package, see r/tools/trim/boost.sh) ship a tiny subset of +# the Boost source archive. +# +# Other configurations keep using Boost's and Apache Thrift's own CMake +# builds. This is not used on Windows yet because the minimal Apache +# Thrift build hasn't been tested there. +if(ARROW_USE_BOOST + AND ARROW_WITH_THRIFT + AND NOT ARROW_BOOST_REQUIRE_LIBRARY + AND NOT ARROW_GANDIVA + AND NOT ARROW_TESTING + AND ARROW_USE_NATIVE_INT128 + AND ARROW_ENABLE_THREADING + AND NOT WIN32) + set(ARROW_BOOST_FOR_THRIFT_ONLY TRUE) +else() + set(ARROW_BOOST_FOR_THRIFT_ONLY FALSE) +endif() + if(ARROW_USE_BOOST) if(ARROW_BOOST_USE_SHARED) # Find shared Boost libraries. @@ -1790,6 +1870,82 @@ endif() # ---------------------------------------------------------------------- # Thrift +# Build only the parts of the Apache Thrift C++ library that Apache +# Parquet uses (TCompactProtocol and TMemoryBuffer), without Apache +# Thrift's CMake build. Apache Thrift's CMake build requires Boost's +# CMake package and builds many sources that need more of Boost +# (e.g. Boost.Locale for TJSONProtocol). See ARROW_BOOST_FOR_THRIFT_ONLY. +# +# This is a macro to be used in build_thrift(). +macro(add_thrift_minimal_library) + fetchcontent_declare(thrift + ${FC_DECLARE_COMMON_OPTIONS} + URL ${THRIFT_SOURCE_URL} + URL_HASH "SHA256=${ARROW_THRIFT_BUILD_SHA256_CHECKSUM}" + # Only extract. We don't use Apache Thrift's + # CMake build. + SOURCE_SUBDIR "arrow-no-cmake") + prepare_fetchcontent() + fetchcontent_makeavailable(thrift) + + set(THRIFT_CPP_SOURCE_DIR "${thrift_SOURCE_DIR}/lib/cpp/src") + # If you update Apache Thrift, check whether Apache Parquet needs + # more sources (undefined symbols when linking libparquet users). + add_library(thrift STATIC + "${THRIFT_CPP_SOURCE_DIR}/thrift/TApplicationException.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/TOutput.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/TUuid.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/protocol/TProtocol.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/transport/TBufferTransports.cpp" + "${THRIFT_CPP_SOURCE_DIR}/thrift/transport/TTransportException.cpp") + set_target_properties(thrift PROPERTIES UNITY_BUILD OFF) + # Third-party code: don't fail on warnings. + target_compile_options(thrift PRIVATE $<$>:-Wno-error>) + target_compile_definitions(thrift PUBLIC THRIFT_STATIC_DEFINE) + target_include_directories(thrift SYSTEM + PUBLIC $ + $) + if(BOOST_VENDORED) + target_link_libraries(thrift PUBLIC $) + else() + target_link_libraries(thrift PUBLIC Boost::headers) + endif() + if(TARGET Threads::Threads) + target_link_libraries(thrift PUBLIC Threads::Threads) + endif() + + # Generate thrift/config.h from Apache Thrift's template, like + # Apache Thrift's build/cmake/ConfigureChecks.cmake does, but only + # with the checks that matter for the sources above and the headers + # they (and Apache Parquet) include. + include(CheckCXXSourceCompiles) + include(CheckIncludeFileCXX) + include(CheckSymbolExists) + check_include_file_cxx(inttypes.h ARROW_THRIFT_HAVE_INTTYPES_H) + check_include_file_cxx(netinet/in.h ARROW_THRIFT_HAVE_NETINET_IN_H) + check_include_file_cxx(stdint.h ARROW_THRIFT_HAVE_STDINT_H) + check_include_file_cxx(sys/param.h ARROW_THRIFT_HAVE_SYS_PARAM_H) + check_symbol_exists(strerror_r "string.h" ARROW_THRIFT_HAVE_STRERROR_R) + # GNU strerror_r() returns char * instead of int. + check_cxx_source_compiles("#include +int main() { char b; char* a = strerror_r(0, &b, 0); static_cast(a); }" + ARROW_THRIFT_STRERROR_R_CHAR_P) + foreach(name + HAVE_INTTYPES_H + HAVE_NETINET_IN_H + HAVE_STDINT_H + HAVE_SYS_PARAM_H + HAVE_STRERROR_R + STRERROR_R_CHAR_P) + set(${name} ${ARROW_THRIFT_${name}}) + endforeach() + set(PACKAGE_NAME "thrift") + set(PACKAGE_VERSION "${ARROW_THRIFT_BUILD_VERSION}") + set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") + configure_file("${thrift_SOURCE_DIR}/build/cmake/config.h.in" + "${thrift_BINARY_DIR}/thrift/config.h") +endmacro() + function(build_thrift) list(APPEND CMAKE_MESSAGE_INDENT "Thrift: ") message(STATUS "Building from source") @@ -1797,66 +1953,70 @@ function(build_thrift) if(CMAKE_VERSION VERSION_LESS 3.26) message(FATAL_ERROR "Require CMake 3.26 or later for building bundled Apache Thrift") endif() - fetchcontent_declare(thrift - ${FC_DECLARE_COMMON_OPTIONS} - URL ${THRIFT_SOURCE_URL} - URL_HASH "SHA256=${ARROW_THRIFT_BUILD_SHA256_CHECKSUM}") + if(ARROW_BOOST_FOR_THRIFT_ONLY) + add_thrift_minimal_library() + else() + fetchcontent_declare(thrift + ${FC_DECLARE_COMMON_OPTIONS} + URL ${THRIFT_SOURCE_URL} + URL_HASH "SHA256=${ARROW_THRIFT_BUILD_SHA256_CHECKSUM}") - prepare_fetchcontent() - set(BUILD_COMPILER OFF) - set(BUILD_EXAMPLES OFF) - set(BUILD_TUTORIALS OFF) - set(CMAKE_UNITY_BUILD OFF) - set(WITH_AS3 OFF) - set(WITH_CPP ON) - set(WITH_C_GLIB OFF) - set(WITH_JAVA OFF) - set(WITH_JAVASCRIPT OFF) - set(WITH_LIBEVENT OFF) - if(MSVC) - if(ARROW_USE_STATIC_CRT) - set(WITH_MT ON) - else() - set(WITH_MT OFF) + prepare_fetchcontent() + set(BUILD_COMPILER OFF) + set(BUILD_EXAMPLES OFF) + set(BUILD_TUTORIALS OFF) + set(CMAKE_UNITY_BUILD OFF) + set(WITH_AS3 OFF) + set(WITH_CPP ON) + set(WITH_C_GLIB OFF) + set(WITH_JAVA OFF) + set(WITH_JAVASCRIPT OFF) + set(WITH_LIBEVENT OFF) + if(MSVC) + if(ARROW_USE_STATIC_CRT) + set(WITH_MT ON) + else() + set(WITH_MT OFF) + endif() endif() - endif() - set(WITH_NODEJS OFF) - set(WITH_OPENSSL OFF) - set(WITH_PYTHON OFF) - set(WITH_QT5 OFF) - set(WITH_ZLIB OFF) - - # Apache Thrift may change CMAKE_DEBUG_POSTFIX. So we'll restore the - # original CMAKE_DEBUG_POSTFIX later. - set(CMAKE_DEBUG_POSTFIX_KEEP ${CMAKE_DEBUG_POSTFIX}) - - # Remove Apache Arrow's CMAKE_MODULE_PATH to ensure using Apache - # Thrift's cmake_modules/. - # - # We can remove this once https://github.com/apache/thrift/pull/3176 - # is merged. - list(POP_FRONT CMAKE_MODULE_PATH) - fetchcontent_makeavailable(thrift) + set(WITH_NODEJS OFF) + set(WITH_OPENSSL OFF) + set(WITH_PYTHON OFF) + set(WITH_QT5 OFF) + set(WITH_ZLIB OFF) + + # Apache Thrift may change CMAKE_DEBUG_POSTFIX. So we'll restore the + # original CMAKE_DEBUG_POSTFIX later. + set(CMAKE_DEBUG_POSTFIX_KEEP ${CMAKE_DEBUG_POSTFIX}) + + # Remove Apache Arrow's CMAKE_MODULE_PATH to ensure using Apache + # Thrift's cmake_modules/. + # + # We can remove this once https://github.com/apache/thrift/pull/3176 + # is merged. + list(POP_FRONT CMAKE_MODULE_PATH) + fetchcontent_makeavailable(thrift) - # Apache Thrift may change CMAKE_DEBUG_POSTFIX. So we restore - # CMAKE_DEBUG_POSTFIX. - set(CMAKE_DEBUG_POSTFIX - ${CMAKE_DEBUG_POSTFIX_KEEP} - CACHE BOOL "" FORCE) + # Apache Thrift may change CMAKE_DEBUG_POSTFIX. So we restore + # CMAKE_DEBUG_POSTFIX. + set(CMAKE_DEBUG_POSTFIX + ${CMAKE_DEBUG_POSTFIX_KEEP} + CACHE BOOL "" FORCE) - if(CMAKE_VERSION VERSION_LESS 3.28) - set_property(DIRECTORY ${thrift_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL TRUE) - endif() + if(CMAKE_VERSION VERSION_LESS 3.28) + set_property(DIRECTORY ${thrift_SOURCE_DIR} PROPERTY EXCLUDE_FROM_ALL TRUE) + endif() - target_include_directories(thrift - INTERFACE $ - $ - ) - if(BOOST_VENDORED) - target_link_libraries(thrift PUBLIC $) - target_link_libraries(thrift PRIVATE $) - else() - target_link_libraries(thrift INTERFACE Boost::headers) + target_include_directories(thrift + INTERFACE $ + $ + ) + if(BOOST_VENDORED) + target_link_libraries(thrift PUBLIC $) + target_link_libraries(thrift PRIVATE $) + else() + target_link_libraries(thrift INTERFACE Boost::headers) + endif() endif() add_library(thrift::thrift INTERFACE IMPORTED) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 3c5518a9596e..ef6e1a4f88dd 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -97,6 +97,7 @@ r/tests/testthat/_snaps/*.md r/inst/include/cpp11.hpp r/inst/include/cpp11/*.hpp r/tools/nixlibs-allowlist.txt +r/inst/COPYRIGHTS .gitattributes ruby/red-arrow/.yardopts ruby/red-arrow-format/lib/arrow-format/org/* diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index d9351d3afe0a..8ea23b6d93b4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -633,14 +633,30 @@ tasks: template: r/github.linux.offline.build.yml # Simulate a CRAN builder without network access: the build should detect - # that it is offline and fall back to a minimal build without downloading - # anything. - test-r-offline-minimal: - ci: github - template: docker-tests/github.linux.yml - params: - flags: '-e ARROW_R_TEST_OFFLINE=true -e NOT_CRAN=false -e SKIP_VIGNETTES=true' + # that it is offline and build libarrow without downloading anything, using + # the thirdparty sources vendored in the package (tools/vendored) to still + # include Parquet, Dataset, JSON, the compression codecs, etc. + # ARROW_SOURCE_HOME='' makes it use the C++ source copied into tools/cpp + # (like the CRAN package does), whose versions.txt has the checksums of the + # trimmed tarballs. The codecs that are off by default on Linux are turned on + # so that every vendored tarball is built. + test-r-offline-vendored: + ci: github + template: docker-tests/github.linux.yml + params: + flags: >- + -e ARROW_R_TEST_OFFLINE=true + -e NOT_CRAN=false + -e SKIP_VIGNETTES=true + -e ARROW_SOURCE_HOME='' + -e ARROW_WITH_BROTLI=ON + -e ARROW_WITH_BZ2=ON + -e ARROW_WITH_ZSTD=ON + -e ARROW_R_EXPECTED_FEATURES=dataset,parquet,json,mimalloc,re2,utf8proc,snappy,gzip,brotli,zstd,lz4,bz2 image: r + # A full libarrow build with 2 cores (NOT_CRAN=false) and no sccache + # (it's offline) + timeout: 180 # This one has -flto=auto test-r-ubuntu-22.04: diff --git a/r/.Rbuildignore b/r/.Rbuildignore index deaf5dd3948c..faae1aaa4480 100644 --- a/r/.Rbuildignore +++ b/r/.Rbuildignore @@ -32,3 +32,5 @@ STYLE.md ^inst/__pycache__$ ^bootstrap.R$ air.toml +^tools/vendored-downloads$ +^tools/trim$ diff --git a/r/.gitignore b/r/.gitignore index a1295ea4e0b5..e16bc75f459b 100644 --- a/r/.gitignore +++ b/r/.gitignore @@ -35,3 +35,5 @@ extra-tests/files # Source tarballs of required dependencies that we ship for offline builds. # They're downloaded by `make sync-cpp` (see tools/vendor_dependencies.sh). /tools/vendored/ +# Cache of the upstream (untrimmed) tarballs used by tools/vendor_dependencies.sh +/tools/vendored-downloads/ diff --git a/r/DESCRIPTION b/r/DESCRIPTION index 1e5733c37cd5..19918beab685 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -24,6 +24,7 @@ Description: 'Apache' 'Arrow' is a cross-language package provides an interface to the 'Arrow C++' library. Depends: R (>= 4.2) License: Apache License (>= 2.0) +Copyright: See file inst/COPYRIGHTS URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/ BugReports: https://github.com/apache/arrow/issues Encoding: UTF-8 diff --git a/r/inst/COPYRIGHTS b/r/inst/COPYRIGHTS new file mode 100644 index 000000000000..779b8b2a5fa8 --- /dev/null +++ b/r/inst/COPYRIGHTS @@ -0,0 +1,78 @@ +The arrow R package is Copyright The Apache Software Foundation and is licensed +under the Apache License, Version 2.0. The Arrow C++ library source copied into +tools/cpp includes code from other projects; see tools/LICENSE.txt and +tools/NOTICE.txt in the source package. + +To build the Arrow C++ library without network access, the source package also +includes (trimmed) source tarballs of these thirdparty dependencies in +tools/vendored. They are only used when the dependency isn't found on the +system, and are statically linked into the package when used. Each tarball +contains the dependency's full license text. + +Boost (headers used by Apache Thrift) + Copyright the Boost authors + Boost Software License 1.0 + +Brotli + Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors + MIT License + +bzip2 + Copyright (C) 1996-2019 Julian R Seward + BSD-style license (see LICENSE in the tarball) + +LZ4 (lib/ only) + Copyright (c) 2011-2020, Yann Collet + BSD 2-Clause License + +mimalloc + Copyright (c) 2018-2025 Microsoft Corporation, Daan Leijen + MIT License + +RapidJSON + Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip + MIT License (with code under the BSD and JSON licenses, see license.txt in + the tarball) + +RE2 + Copyright (c) 2009 The RE2 Authors + BSD 3-Clause License + +simdjson + Copyright 2018-2025 The simdjson authors + Apache License 2.0 or MIT License + +Snappy + Copyright 2011, Google Inc. + BSD 3-Clause License + +Apache Thrift + Copyright (C) 2006-2019, The Apache Software Foundation + Apache License 2.0 + +uriparser (library sources only) + Copyright (C) 2007, Weijia Song and Sebastian Pipping + BSD 3-Clause License + +utf8proc + Copyright (c) 2014-2021 Steven G. Johnson, Jiahao Chen, Tony Kelman, + Jonas Fonseca, and other contributors + Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany + MIT License, with Unicode data under the Unicode Data Files and Software + License + +xsimd + Copyright (c) 2016, Johan Mabille, Sylvain Corlay, Wolf Vollprecht and + Martin Renou + Copyright (c) 2016, QuantStack + Copyright (c) 2018, Serge Guelton + BSD 3-Clause License + +zlib + Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler + zlib License + +Zstandard (zstd) + Copyright (c) Meta Platforms, Inc. and affiliates + BSD 3-Clause License (zstd is dual-licensed BSD/GPLv2; it is used under the + BSD license) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 226894372ec5..a2c321647bd5 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -486,7 +486,7 @@ find_local_source <- function() { # The second probably occurs if we're installing the arrow R package cpp_dir_options <- c( file.path(Sys.getenv("ARROW_SOURCE_HOME", ".."), "cpp"), - "tools/cpp" + package_cpp_dir ) for (cpp_dir in cpp_dir_options) { if (file.exists(file.path(cpp_dir, "src/arrow/api.h"))) { @@ -625,30 +625,35 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- with_cloud_support(env_var_list) env_var_list <- with_wasm_support(env_var_list) - # turn_off_all_optional_features() needs to happen after - # with_cloud_support(), since it might turn features ON. + # turn_off_all_optional_features() and turn_off_unvendored_features() need to + # happen after with_cloud_support(), since it might turn features ON. thirdparty_deps_unavailable <- !download_ok && !dir.exists(thirdparty_dependency_dir) && !env_is("ARROW_DEPENDENCY_SOURCE", "system") do_minimal_build <- env_is("LIBARROW_MINIMAL", "true") + # Thirdparty source tarballs shipped in the package + vendored <- usable_vendored_dependencies(src_dir) if (do_minimal_build) { env_var_list <- turn_off_all_optional_features(env_var_list) } else if (thirdparty_deps_unavailable) { cat(paste0( "*** Building C++ library from source, but downloading thirdparty dependencies\n", - " is not possible, so this build will turn off all thirdparty features.\n", + " is not possible, so this build will turn off the thirdparty features\n", + " whose sources are not included in the package.\n", " See installation guide for details:\n", " https://arrow.apache.org/docs/r/articles/install.html\n" )) - env_var_list <- turn_off_all_optional_features(env_var_list) + env_var_list <- turn_off_unvendored_features(env_var_list, names(vendored)) } else if (dir.exists(thirdparty_dependency_dir)) { # Add the *_SOURCE_URL env vars env_var_list <- set_thirdparty_urls(env_var_list) } - # Some thirdparty dependencies are required even for a minimal build, so they - # can't be turned off like the ones above. - env_var_list <- with_vendored_dependencies(env_var_list) + # Point the build at the vendored sources of the thirdparty dependencies. + # Some of them are required even for a minimal build, so they can't be turned + # off like the ones above. + env_var_list <- with_vendored_dependencies(env_var_list, vendored) + use_vendored_checksums(src_dir, env_var_list, vendored) env_vars <- env_vars_as_string(env_var_list) lg("arrow %s", ifelse(quietly, "", paste("with", env_vars)), .indent = "****") @@ -795,7 +800,9 @@ cmake_version <- function(cmd = "cmake") { ) } -turn_off_all_optional_features <- function(env_var_list) { +# `keep` is a vector of features (names in `turn_off` below, or +# "ARROW_SIMD_LEVEL") to leave as they are. +turn_off_all_optional_features <- function(env_var_list, keep = character()) { # Because these are done as environment variables (as opposed to build flags), # setting these to "OFF" overrides any previous setting. We don't need to # check the existing value. @@ -822,19 +829,53 @@ turn_off_all_optional_features <- function(env_var_list) { "ARROW_WITH_ZLIB" = "OFF", "ARROW_WITH_ZSTD" = "OFF", "ARROW_WITH_RE2" = "OFF", - "ARROW_WITH_UTF8PROC" = "OFF", + "ARROW_WITH_UTF8PROC" = "OFF" + ) + turn_off <- turn_off[setdiff(names(turn_off), keep)] + if (!("ARROW_SIMD_LEVEL" %in% keep)) { # The syntax to turn off XSIMD is different. # Pull existing value of EXTRA_CMAKE_FLAGS first (must be defined) - "EXTRA_CMAKE_FLAGS" = paste( - env_var_list[["EXTRA_CMAKE_FLAGS"]], - "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE" + turn_off <- c( + turn_off, + "EXTRA_CMAKE_FLAGS" = paste( + env_var_list[["EXTRA_CMAKE_FLAGS"]], + "-DARROW_SIMD_LEVEL=NONE -DARROW_RUNTIME_SIMD_LEVEL=NONE" + ) ) - ) + } # Create a new env_var_list, with the values of turn_off set. # replace() also adds new values if they didn't exist before replace(env_var_list, names(turn_off), turn_off) } +# Without network access, libarrow can only build the thirdparty dependencies +# whose source tarballs are shipped in the package (see +# `with_vendored_dependencies()`), so turn off every optional feature except +# those whose dependencies are all vendored. Features that are kept are left as +# they are (e.g. as the user or the platform defaults set them), not turned on. +# `available` is the ARROW_*_URL names of the usable vendored tarballs (see +# `usable_vendored_dependencies()`). +turn_off_unvendored_features <- function(env_var_list, available) { + has_all <- vapply(vendored_features, function(deps) all(deps %in% available), logical(1)) + keep <- names(vendored_features)[has_all] + if ("ARROW_PARQUET" %in% keep) { + # Dataset needs no thirdparty dependencies itself, but it's only useful + # with Parquet (and we've always turned them on and off together) + keep <- c(keep, "ARROW_DATASET") + } + if ("ARROW_XSIMD_URL" %in% available) { + keep <- c(keep, "ARROW_SIMD_LEVEL") + } + if (!quietly) { + lg( + "Features with vendored dependencies: %s", + if (length(keep)) paste(keep, collapse = ", ") else "none", + .indent = "****" + ) + } + turn_off_all_optional_features(env_var_list, keep = keep) +} + get_component_names <- function() { if (!isTRUE(Sys.which("bash") != "")) { stop("nixlibs.R requires bash to be installed and available in your PATH") @@ -874,26 +915,53 @@ set_thirdparty_urls <- function(env_var_list) { env_var_list } -# libarrow requires some thirdparty dependencies, even for a minimal build. -# The libarrow build uses them from the system if they are available and -# otherwise builds them from source, downloading from ARROW_*_URL. To make sure -# that works without network access, `make sync-cpp` saves their source -# tarballs into vendored_dependency_dir and we point ARROW_*_URL at them here. +# The libarrow build uses thirdparty dependencies from the system if they are +# available and otherwise builds them from source, downloading from ARROW_*_URL. +# Some are required even for a minimal build. To make sure that works without +# network access, `make sync-cpp` saves (trimmed) source tarballs into +# vendored_dependency_dir and we point ARROW_*_URL at them here. # This does not change whether the system versions are used. -with_vendored_dependencies <- function(env_var_list) { - if (!dir.exists(vendored_dependency_dir)) { +# `vendored` is the result of `usable_vendored_dependencies()`. +with_vendored_dependencies <- function(env_var_list, vendored) { + for (env_varname in names(vendored)) { + if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { + # Respect the user's (or set_thirdparty_urls()'s) choice + next + } + env_var_list <- replace(env_var_list, env_varname, vendored[[env_varname]]) + } + env_var_list +} + +# Returns the paths of the vendored tarballs that the libarrow source in +# `src_dir` can use, named by their ARROW_*_URL. +# The tarballs are trimmed, so the libarrow build only accepts them once +# `use_vendored_checksums()` has put their checksums in versions.txt. We only +# do that to the package's own copy of the source (tools/cpp), not e.g. ../cpp +# of a git checkout, and only for tarballs that were trimmed from the version +# that versions.txt pins (according to the checksums.txt file that +# tools/vendor_dependencies.sh writes). +usable_vendored_dependencies <- function(src_dir) { + manifest <- read_vendored_checksums() + if (is.null(manifest)) { # e.g. building from a git checkout without `make sync-cpp` if (!quietly) { lg("No vendored dependencies found, they will be built with CMake if needed", .indent = "****") } - return(env_var_list) + return(character()) } + is_package_cpp_dir <- !is.null(src_dir) && + identical(normalizePath(src_dir, mustWork = FALSE), normalizePath(package_cpp_dir, mustWork = FALSE)) + if (!is_package_cpp_dir) { + if (!quietly) { + lg("Not using vendored dependencies, they are only used to build %s", package_cpp_dir, .indent = "****") + } + return(character()) + } + expected <- read_thirdparty_checksums(file.path(src_dir, "thirdparty", "versions.txt")) deps_df <- get_component_names() + usable <- character() for (env_varname in vendored_dependencies) { - if (!is.null(env_var_list[[env_varname]]) || !env_is(env_varname, "")) { - # Respect the user's (or set_thirdparty_urls()'s) choice - next - } filename <- deps_df$filename[deps_df$env_varname == env_varname] tarball <- file.path(vendored_dependency_dir, filename) if (length(tarball) != 1 || !file.exists(tarball)) { @@ -902,9 +970,68 @@ with_vendored_dependencies <- function(env_var_list) { } next } - env_var_list <- replace(env_var_list, env_varname, normalizePath(tarball)) + checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) + entry <- manifest[manifest$filename == filename, , drop = FALSE] + # versions.txt has the vendored checksum if use_vendored_checksums() has + # already run on it (e.g. when installing from the r/ directory again) + matches <- nrow(entry) == 1 && + unname(expected[checksum_varname]) %in% c(entry$upstream_sha256, entry$vendored_sha256) + if (!matches) { + if (!quietly) { + lg( + "Vendored source for %s doesn't match %s, it will be downloaded if needed", + env_varname, + file.path(src_dir, "thirdparty", "versions.txt"), + .indent = "****" + ) + } + next + } + usable[[env_varname]] <- normalizePath(tarball) } - env_var_list + usable +} + +# Read the checksums.txt that tools/vendor_dependencies.sh writes, with the +# checksums of each vendored tarball before (upstream) and after trimming +read_vendored_checksums <- function() { + manifest_file <- file.path(vendored_dependency_dir, "checksums.txt") + if (!file.exists(manifest_file)) { + return(NULL) + } + read.table(manifest_file, header = TRUE, stringsAsFactors = FALSE) +} + +# The libarrow build checks the checksums of the thirdparty tarballs against +# src_dir's versions.txt, so replace the upstream checksums of the vendored +# tarballs that `env_var_list` points at with the trimmed tarballs' checksums. +use_vendored_checksums <- function(src_dir, env_var_list, vendored) { + used <- vendored[vapply( + names(vendored), + function(env_varname) identical(env_var_list[[env_varname]], vendored[[env_varname]]), + logical(1) + )] + if (length(used) == 0) { + return(invisible()) + } + manifest <- read_vendored_checksums() + versions_txt <- file.path(src_dir, "thirdparty", "versions.txt") + lines <- readLines(versions_txt) + for (env_varname in names(used)) { + checksum_varname <- sub("_URL$", "_BUILD_SHA256_CHECKSUM", env_varname) + checksum <- manifest$vendored_sha256[manifest$filename == basename(used[[env_varname]])] + lines <- sub(paste0("^", checksum_varname, "=.*$"), paste0(checksum_varname, "=", checksum), lines) + } + writeLines(lines, versions_txt) +} + +# Parse the *_BUILD_SHA256_CHECKSUM=... lines of a versions.txt into a named vector +read_thirdparty_checksums <- function(versions_txt) { + if (!file.exists(versions_txt)) { + return(character()) + } + lines <- grep("^[A-Z0-9_]+_BUILD_SHA256_CHECKSUM=", readLines(versions_txt), value = TRUE) + setNames(sub("^[^=]*=", "", lines), sub("=.*$", "", lines)) } # this is generally about features that people asked for via environment variables, but @@ -1140,11 +1267,37 @@ if (download_libarrow_ok && !is_release && !test_mode) { # `create_package_with_all_dependencies()` in install-arrow.R thirdparty_dependency_dir <- Sys.getenv("ARROW_THIRDPARTY_DEPENDENCY_DIR", "tools/thirdparty_dependencies") -# Source tarballs of dependencies that libarrow requires even for a minimal build. -# These are always shipped in the package (see `with_vendored_dependencies()`). -# If you change this path or list, you also need to edit tools/vendor_dependencies.sh +# Trimmed source tarballs of thirdparty dependencies, shipped in the package so +# that building without network access still works and can include the features +# below (see `with_vendored_dependencies()` and `turn_off_unvendored_features()`). +# If you change this path or these lists, you also need to edit +# tools/vendor_dependencies.sh vendored_dependency_dir <- "tools/vendored" -vendored_dependencies <- c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL") +# The copy of the libarrow source in the package (see `make sync-cpp`) +package_cpp_dir <- "tools/cpp" +# Optional features and the ARROW_*_URL of each dependency they need (when the +# dependency is built from source). See "Some EP's require other EP's" in +# cpp/cmake_modules/ThirdpartyToolchain.cmake +vendored_features <- list( + # Parquet turns on Thrift and simdjson; Thrift needs zlib and Boost + ARROW_PARQUET = c("ARROW_BOOST_URL", "ARROW_SIMDJSON_URL", "ARROW_THRIFT_URL", "ARROW_ZLIB_URL"), + ARROW_JSON = c("ARROW_RAPIDJSON_URL", "ARROW_SIMDJSON_URL"), + ARROW_MIMALLOC = "ARROW_MIMALLOC_URL", + ARROW_WITH_BROTLI = "ARROW_BROTLI_URL", + ARROW_WITH_BZ2 = "ARROW_BZIP2_URL", + ARROW_WITH_LZ4 = "ARROW_LZ4_URL", + ARROW_WITH_RE2 = "ARROW_RE2_URL", + ARROW_WITH_SNAPPY = "ARROW_SNAPPY_URL", + ARROW_WITH_UTF8PROC = "ARROW_UTF8PROC_URL", + ARROW_WITH_ZLIB = "ARROW_ZLIB_URL", + ARROW_WITH_ZSTD = "ARROW_ZSTD_URL" +) +# uriparser and xsimd are required even for a minimal build +vendored_dependencies <- unique(c( + "ARROW_URIPARSER_URL", + "ARROW_XSIMD_URL", + unlist(vendored_features, use.names = FALSE) +)) arrow_versioned <- paste0("arrow-", VERSION) # configure.win uses a different libarrow dir and the zip is already nested diff --git a/r/tools/test-nixlibs.R b/r/tools/test-nixlibs.R index 9d8abccfbb5e..0320658d41d4 100644 --- a/r/tools/test-nixlibs.R +++ b/r/tools/test-nixlibs.R @@ -233,58 +233,213 @@ test_that("find_latest_nightly()", { ) }) -test_that("with_vendored_dependencies()", { - vendored_dir <- tempfile() - dir.create(vendored_dir) - on.exit(unlink(vendored_dir, recursive = TRUE), add = TRUE) - original_dir <- nixlibs_env$vendored_dependency_dir - original_get_component_names <- nixlibs_env$get_component_names - original_quietly <- nixlibs_env$quietly - on.exit(nixlibs_env$vendored_dependency_dir <- original_dir, add = TRUE) - on.exit(nixlibs_env$get_component_names <- original_get_component_names, add = TRUE) - on.exit(nixlibs_env$quietly <- original_quietly, add = TRUE) - nixlibs_env$vendored_dependency_dir <- vendored_dir - nixlibs_env$get_component_names <- function() { - data.frame( - env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL"), - filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz") +# Set up a fake vendored_dependency_dir and package_cpp_dir for the vendored +# dependency tests. `vendored` is a named vector of filename = upstream +# checksum in checksums.txt (the trimmed checksum is "-trimmed"), +# `expected` is the checksums in the source dir's versions.txt, named by +# ARROW_*_URL. +local_vendored_dependencies <- function(vendored, expected, env = parent.frame()) { + vendored_dir <- withr::local_tempdir(.local_envir = env) + src_dir <- withr::local_tempdir(.local_envir = env) + for (filename in names(vendored)) { + file.create(file.path(vendored_dir, filename)) + } + if (length(vendored)) { + writeLines( + c( + "filename upstream_sha256 vendored_sha256", + paste(names(vendored), vendored, paste0(vendored, "-trimmed")) + ), + file.path(vendored_dir, "checksums.txt") ) } - withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) + dir.create(file.path(src_dir, "thirdparty")) + writeLines( + c( + "# a comment", + paste0(sub("_URL$", "_BUILD_SHA256_CHECKSUM", names(expected)), "=", expected) + ), + file.path(src_dir, "thirdparty", "versions.txt") + ) + local_binding("vendored_dependency_dir", vendored_dir, env) + local_binding("package_cpp_dir", src_dir, env) + local_binding( + "get_component_names", + function() { + data.frame( + env_varname = c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL", "ARROW_JEMALLOC_URL"), + filename = c("uriparser-1.0.2.tar.bz2", "xsimd-14.3.0.tar.gz", "zstd-1.5.7.tar.gz", "jemalloc-5.3.0.tar.bz2") + ) + }, + env + ) + list(vendored_dir = vendored_dir, src_dir = src_dir) +} + +# Temporarily replace a variable in nixlibs_env +local_binding <- function(name, value, env = parent.frame()) { + original <- nixlibs_env[[name]] + assign(name, value, envir = nixlibs_env) + withr::defer(assign(name, original, envir = nixlibs_env), envir = env) +} - # Without the vendored tarballs, nothing is set - nixlibs_env$quietly <- TRUE - expect_silent( - expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")) +test_that("usable_vendored_dependencies()", { + local_binding("quietly", TRUE) + # Nothing vendored + dirs <- local_vendored_dependencies(character(), c(ARROW_URIPARSER_URL = "aaa")) + expect_identical(usable_vendored_dependencies(dirs$src_dir), character()) + + dirs <- local_vendored_dependencies( + c( + "uriparser-1.0.2.tar.bz2" = "aaa", + "xsimd-14.3.0.tar.gz" = "bbb", + "zstd-1.5.7.tar.gz" = "ccc", + # Not in vendored_dependencies, so it isn't used + "jemalloc-5.3.0.tar.bz2" = "ddd" + ), + c( + ARROW_URIPARSER_URL = "aaa", + # A different version than the vendored one + ARROW_XSIMD_URL = "other", + # Already has the trimmed checksum (use_vendored_checksums() has run) + ARROW_ZSTD_URL = "ccc-trimmed", + ARROW_JEMALLOC_URL = "ddd" + ) + ) + expect_identical( + usable_vendored_dependencies(dirs$src_dir), + c( + ARROW_URIPARSER_URL = normalizePath(file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2")), + ARROW_ZSTD_URL = normalizePath(file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz")) + ) + ) + # Mismatches are only reported with verbose logging + local_binding("quietly", FALSE) + expect_output( + usable_vendored_dependencies(dirs$src_dir), + "Vendored source for ARROW_XSIMD_URL doesn't match" + ) + + # Only used to build package_cpp_dir (not e.g. ../cpp of a git checkout) + other_src_dir <- withr::local_tempdir() + file.copy(file.path(dirs$src_dir, "thirdparty"), other_src_dir, recursive = TRUE) + expect_output( + expect_identical(usable_vendored_dependencies(other_src_dir), character()), + "Not using vendored dependencies" + ) + expect_output( + expect_identical(usable_vendored_dependencies(NULL), character()), + "Not using vendored dependencies" ) - # The missing tarballs are only reported with verbose logging - nixlibs_env$quietly <- FALSE + + # A missing tarball isn't used, even if it's in checksums.txt + unlink(file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz")) expect_output( - expect_identical(with_vendored_dependencies(list(A = "1")), list(A = "1")), - "Vendored source for ARROW_URIPARSER_URL not found" + expect_named(usable_vendored_dependencies(dirs$src_dir), "ARROW_URIPARSER_URL"), + "Vendored source for ARROW_ZSTD_URL not found" ) - nixlibs_env$quietly <- TRUE +}) - uriparser <- file.path(vendored_dir, "uriparser-1.0.2.tar.bz2") - xsimd <- file.path(vendored_dir, "xsimd-14.3.0.tar.gz") - # Only dependencies in vendored_dependencies are used - zstd <- file.path(vendored_dir, "zstd-1.5.7.tar.gz") - file.create(uriparser, xsimd, zstd) +test_that("use_vendored_checksums()", { + local_binding("quietly", TRUE) + dirs <- local_vendored_dependencies( + c("uriparser-1.0.2.tar.bz2" = "aaa", "xsimd-14.3.0.tar.gz" = "bbb", "zstd-1.5.7.tar.gz" = "ccc"), + c(ARROW_URIPARSER_URL = "aaa", ARROW_XSIMD_URL = "bbb", ARROW_ZSTD_URL = "ccc", ARROW_JEMALLOC_URL = "ddd") + ) + versions_txt <- file.path(dirs$src_dir, "thirdparty", "versions.txt") + original <- readLines(versions_txt) + vendored <- c( + ARROW_URIPARSER_URL = file.path(dirs$vendored_dir, "uriparser-1.0.2.tar.bz2"), + ARROW_XSIMD_URL = file.path(dirs$vendored_dir, "xsimd-14.3.0.tar.gz"), + ARROW_ZSTD_URL = file.path(dirs$vendored_dir, "zstd-1.5.7.tar.gz") + ) + + # Nothing is pointed at the vendored tarballs + use_vendored_checksums(dirs$src_dir, list(), vendored) + expect_identical(readLines(versions_txt), original) + + # Only the ones that are (e.g. not when the user set ARROW_XSIMD_URL) + env_var_list <- list( + ARROW_URIPARSER_URL = vendored[["ARROW_URIPARSER_URL"]], + ARROW_XSIMD_URL = "https://example.com/xsimd.tar.gz", + ARROW_ZSTD_URL = vendored[["ARROW_ZSTD_URL"]] + ) + use_vendored_checksums(dirs$src_dir, env_var_list, vendored) expect_identical( - with_vendored_dependencies(list(A = "1")), - list(A = "1", ARROW_URIPARSER_URL = normalizePath(uriparser), ARROW_XSIMD_URL = normalizePath(xsimd)) + readLines(versions_txt), + c( + "# a comment", + "ARROW_URIPARSER_BUILD_SHA256_CHECKSUM=aaa-trimmed", + "ARROW_XSIMD_BUILD_SHA256_CHECKSUM=bbb", + "ARROW_ZSTD_BUILD_SHA256_CHECKSUM=ccc-trimmed", + "ARROW_JEMALLOC_BUILD_SHA256_CHECKSUM=ddd" + ) + ) + # and they are still usable afterwards + expect_named( + usable_vendored_dependencies(dirs$src_dir), + c("ARROW_URIPARSER_URL", "ARROW_XSIMD_URL", "ARROW_ZSTD_URL") + ) +}) + +test_that("with_vendored_dependencies()", { + vendored <- c(ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") + withr::local_envvar(c(ARROW_URIPARSER_URL = NA, ARROW_XSIMD_URL = NA)) + + expect_identical(with_vendored_dependencies(list(A = "1"), character()), list(A = "1")) + expect_identical( + with_vendored_dependencies(list(A = "1"), vendored), + list(A = "1", ARROW_URIPARSER_URL = "/vendored/uriparser.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") ) # Already set (e.g. by set_thirdparty_urls()) is respected expect_identical( - with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2")), - list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = normalizePath(xsimd)) + with_vendored_dependencies(list(ARROW_URIPARSER_URL = "other.tar.bz2"), vendored), + list(ARROW_URIPARSER_URL = "other.tar.bz2", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") ) # As is a user-set env var withr::local_envvar(c(ARROW_URIPARSER_URL = "https://example.com/uriparser.tar.bz2")) expect_identical( - with_vendored_dependencies(list(A = "1")), - list(A = "1", ARROW_XSIMD_URL = normalizePath(xsimd)) + with_vendored_dependencies(list(A = "1"), vendored), + list(A = "1", ARROW_XSIMD_URL = "/vendored/xsimd.tar.gz") + ) +}) + +test_that("turn_off_unvendored_features()", { + local_binding("quietly", TRUE) + env_var_list <- list(EXTRA_CMAKE_FLAGS = "", ARROW_WITH_ZSTD = "ON", ARROW_S3 = "ON") + + # Nothing vendored: the same as turn_off_all_optional_features() + expect_identical( + turn_off_unvendored_features(env_var_list, character()), + turn_off_all_optional_features(env_var_list) + ) + + # Everything vendored: only features that need unvendored dependencies are + # turned off, and the rest are left as they were (not turned on) + result <- turn_off_unvendored_features(env_var_list, vendored_dependencies) + expect_identical(result$ARROW_S3, "OFF") + expect_identical(result$ARROW_JEMALLOC, "OFF") + expect_identical(result$ARROW_WITH_ZSTD, "ON") + for (feature in setdiff(c(names(vendored_features), "ARROW_DATASET"), "ARROW_WITH_ZSTD")) { + expect_null(result[[feature]]) + } + expect_identical(result$EXTRA_CMAKE_FLAGS, "") + + # Parquet (and Dataset) need all of their dependencies + result <- turn_off_unvendored_features( + env_var_list, + c("ARROW_XSIMD_URL", "ARROW_THRIFT_URL", "ARROW_ZSTD_URL") + ) + expect_identical(result$ARROW_PARQUET, "OFF") + expect_identical(result$ARROW_DATASET, "OFF") + expect_identical(result$ARROW_WITH_ZSTD, "ON") + expect_identical(result$ARROW_WITH_SNAPPY, "OFF") + # SIMD is only turned off if xsimd isn't vendored + expect_identical(result$EXTRA_CMAKE_FLAGS, "") + expect_match( + turn_off_unvendored_features(env_var_list, "ARROW_ZSTD_URL")$EXTRA_CMAKE_FLAGS, + "-DARROW_SIMD_LEVEL=NONE" ) }) diff --git a/r/tools/trim/boost.sh b/r/tools/trim/boost.sh new file mode 100644 index 000000000000..bbbeba6dd566 --- /dev/null +++ b/r/tools/trim/boost.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Trim the Boost source archive (boost-X.Y.Z-cmake.tar.gz) to the +# headers that Apache Thrift needs. +# +# The R package only needs Boost for Apache Thrift (for Parquet), and +# Arrow's CMake uses bundled Boost header-only in that case (see +# ARROW_BOOST_FOR_THRIFT_ONLY in cpp/cmake_modules/ThirdpartyToolchain.cmake). +# So we keep only the headers reachable via #include from the Boost +# headers that Apache Thrift includes, and keep the directory layout of +# the upstream archive (boost-X.Y.Z/libs//include/boost/...). +# +# The #include graph is followed without evaluating the preprocessor, +# so headers for all platforms and compilers are kept. boost/config is +# kept entirely because it includes headers via macros. +# +# Usage: tools/trim/boost.sh IN_TARBALL OUT_TARBALL + +set -euo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2 + exit 1 +fi + +# The Boost headers that Apache Thrift's C++ library includes (in the +# sources Arrow builds and in the headers Parquet includes). Update +# this when Apache Thrift is upgraded: +# grep -rh '#include /dev/null | grep -q "GNU tar"; then + wildcards="--wildcards" +fi +tar -xzf "${in}" -C "${work}/src" ${wildcards} \ + '*/libs/*/include/boost/*' \ + '*/LICENSE_1_0.txt' +top="$(ls "${work}/src")" +cd "${work}/src/${top}" + +find libs -type f -path '*/include/boost/*' > "${work}/index" + +# Breadth-first search of the #include graph. Input: the index of all +# headers. Output: the headers to keep. +awk -v seeds="${SEEDS}" ' +function normalize(path, n, parts, i, out, m, stack) { + n = split(path, parts, "/") + m = 0 + for (i = 1; i <= n; i++) { + if (parts[i] == "" || parts[i] == ".") continue + if (parts[i] == "..") { if (m > 0) m--; continue } + stack[++m] = parts[i] + } + out = "" + for (i = 1; i <= m; i++) out = out (i > 1 ? "/" : "") stack[i] + return out +} +{ + # libs/uuid/include/boost/uuid/uuid.hpp -> boost/uuid/uuid.hpp + header = substr($0, index($0, "/include/boost/") + length("/include/")) + path[header] = $0 +} +END { + tail = split(seeds, queue, " ") + head = 0 + while (head < tail) { + header = queue[++head] + if ((header in seen) || !(header in path)) continue + seen[header] = 1 + print path[header] + dir = header + sub(/\/[^\/]*$/, "", dir) + file = path[header] + while ((getline line < file) > 0) { + if (line !~ /^[ \t]*#[ \t]*include[ \t]*[<"]/) continue + sub(/^[ \t]*#[ \t]*include[ \t]*[<"]/, "", line) + sub(/[>"].*$/, "", line) + if (line ~ /^boost\//) { + queue[++tail] = line + } else { + relative = normalize(dir "/" line) + if (relative in path) queue[++tail] = relative + } + } + close(file) + } +}' "${work}/index" > "${work}/keep" + +if ! grep -q "/uuid/uuid.hpp$" "${work}/keep"; then + echo "Failed to find Boost headers for Apache Thrift in ${in}" >&2 + exit 1 +fi + +{ + cat "${work}/keep" + find libs/config/include -type f + echo LICENSE_1_0.txt +} | LC_ALL=C sort -u | sed -e "s|^|${top}/|" > "${work}/files" + +echo "Keeping $(wc -l < "${work}/files" | tr -d ' ') files from Boost libraries:" \ + "$(sed -n -e 's|^[^/]*/libs/\([^/]*\)/.*|\1|p' "${work}/files" | sort -u | tr '\n' ' ')" + +cd "${work}/src" +tar -cf - -T "${work}/files" | gzip -n -9 > "${out}" diff --git a/r/tools/trim/thrift.sh b/r/tools/trim/thrift.sh new file mode 100644 index 000000000000..b71df2fdad13 --- /dev/null +++ b/r/tools/trim/thrift.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Trim the Apache Thrift source archive to what Arrow's minimal +# Apache Thrift build uses: the C++ library sources and the config.h +# template. Arrow doesn't use Apache Thrift's CMake build when Boost is +# only needed for Apache Thrift (see ARROW_BOOST_FOR_THRIFT_ONLY and +# add_thrift_minimal_library() in +# cpp/cmake_modules/ThirdpartyToolchain.cmake), which is always the case +# for the R package. +# +# Usage: tools/trim/thrift.sh IN_TARBALL OUT_TARBALL + +set -euo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 IN_TARBALL OUT_TARBALL" >&2 + exit 1 +fi + +in="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +out="$(cd "$(dirname "$2")" && pwd)/$(basename "$2")" + +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT +tar -xzf "${in}" -C "${work}" +top="$(ls "${work}")" + +cd "${work}" +for path in LICENSE NOTICE build/cmake/config.h.in lib/cpp/src/thrift; do + if [ ! -e "${top}/${path}" ]; then + echo "${top}/${path} not found in ${in}" >&2 + exit 1 + fi +done +find "${top}/LICENSE" "${top}/NOTICE" "${top}/build/cmake/config.h.in" \ + "${top}/lib/cpp/src/thrift" -type f | LC_ALL=C sort > files +tar -cf - -T files | gzip -n -9 > "${out}" diff --git a/r/tools/vendor_dependencies.sh b/r/tools/vendor_dependencies.sh index caa28bff5ce2..686ce981e286 100644 --- a/r/tools/vendor_dependencies.sh +++ b/r/tools/vendor_dependencies.sh @@ -17,30 +17,80 @@ # specific language governing permissions and limitations # under the License. -# libarrow requires some thirdparty dependencies, even for a minimal build. -# This script downloads the source tarballs that libarrow would otherwise -# download at build time, so that they can be shipped in the R package. -# nixlibs.R uses them when the dependencies are not found on the system, so -# that building without network access (e.g. on CRAN) still works. +# libarrow downloads the source of its thirdparty dependencies at build time +# when they aren't found on the system. This script downloads those source +# tarballs ahead of time so that they can be shipped in the R package, which +# lets building without network access (e.g. on CRAN) still include features +# like Parquet. nixlibs.R uses them when the dependencies are not found on the +# system, and turns off the features whose dependencies aren't vendored. +# +# To keep the package small, each tarball is trimmed down to what the libarrow +# build uses (no tests, docs, other language bindings, ...). Trimmed tarballs +# no longer match the checksums pinned in versions.txt, so this script records +# the upstream and trimmed checksums of each tarball in DESTDIR/checksums.txt. +# When building the package's copy of the libarrow source (tools/cpp), nixlibs.R +# uses the tarballs whose upstream checksum matches versions.txt and replaces +# it with the trimmed one. +# +# Trimming is done by, in order of preference: +# 1. tools/trim/.sh IN_TARBALL OUT_TARBALL, if it exists, where +# is the lowercase dependency name (e.g. ARROW_BOOST_URL -> boost). Use +# this for dependencies that need more than a few `rm`s. +# 2. Otherwise, trim_generic() plus the dependency-specific removals in +# trim_dependency() below. # # Usage: tools/vendor_dependencies.sh [DESTDIR] -# Must be run after tools/cpp has been synced (see `make sync-cpp`). +# Must be run after tools/cpp has been synced (see `make sync-cpp`). Needs curl +# and cmake (to handle .tar.bz2 without a bzip2 binary). +# +# Environment variables: +# ARROW_VENDOR_DOWNLOAD_DIR: where to cache the upstream (untrimmed) +# tarballs between runs (default: tools/vendored-downloads, which is +# ignored by git and R CMD build). set -euo pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DESTDIR="${1:-${SOURCE_DIR}/vendored}" +DOWNLOAD_DIR="${ARROW_VENDOR_DOWNLOAD_DIR:-${SOURCE_DIR}/vendored-downloads}" +VERSIONS_TXT="${SOURCE_DIR}/cpp/thirdparty/versions.txt" # If you change this list, you also need to edit -# `vendored_dependencies` in nixlibs.R +# `vendored_dependencies` (and `vendored_features`) in nixlibs.R VENDORED_DEPENDENCIES=( + # Required even for a minimal build ARROW_URIPARSER_URL ARROW_XSIMD_URL + # Parquet (ARROW_PARQUET turns on Thrift and simdjson; Thrift needs zlib and + # Boost headers) + ARROW_BOOST_URL + ARROW_SIMDJSON_URL + ARROW_THRIFT_URL + ARROW_ZLIB_URL + # Compression codecs + ARROW_BROTLI_URL + ARROW_BZIP2_URL + ARROW_LZ4_URL + ARROW_SNAPPY_URL + ARROW_ZSTD_URL + # Compute (RE2, utf8proc), JSON (RapidJSON, simdjson) and the allocator + ARROW_MIMALLOC_URL + ARROW_RAPIDJSON_URL + ARROW_RE2_URL + ARROW_UTF8PROC_URL ) +if [ ! -f "${VERSIONS_TXT}" ]; then + echo "${VERSIONS_TXT} not found. Run this from \`make sync-cpp\`." >&2 + exit 1 +fi + # Load `DEPENDENCIES` and the *_BUILD_SHA256_CHECKSUM variables. # shellcheck source=/dev/null -source "${SOURCE_DIR}/cpp/thirdparty/versions.txt" +source "${VERSIONS_TXT}" + +# Don't add macOS metadata (._* files, extended attributes) to the tarballs +export COPYFILE_DISABLE=1 sha256() { if command -v sha256sum > /dev/null 2>&1; then @@ -50,8 +100,150 @@ sha256() { fi } -mkdir -p "${DESTDIR}" -keep=() +# rm -rf the given paths, relative to the current directory +rmd() { + rm -rf "$@" +} + +# Remove everything in the current directory except the given names +keep_only() { + local entry name keep + for entry in * .[!.]*; do + [ -e "${entry}" ] || continue + keep=false + for name in "$@"; do + if [ "${entry}" = "${name}" ]; then + keep=true + fi + done + if [ "${keep}" = "false" ]; then + rm -rf "${entry}" + fi + done +} + +# Remove directories that no bundled build of our dependencies compiles. +# Arguments are paths (relative to the source root) to keep anyway. +trim_generic() { + local dir keep k + find . -depth -type d \( -name test -o -name tests -o -name testdata \ + -o -name doc -o -name docs -o -name example -o -name examples \ + -o -name benchmark -o -name benchmarks -o -name bench -o -name fuzz \) \ + -print | while read -r dir; do + keep=false + for k in "$@"; do + if [ "${dir}" = "./${k}" ]; then + keep=true + fi + done + if [ "${keep}" = "false" ]; then + rm -rf "${dir}" + fi + done +} + +# Trim an extracted source tree (the current directory) for the dependency +# named $1 (e.g. "thrift") +trim_dependency() { + case "$1" in + brotli) + rmd go python java csharp js research + # install() copies the man pages from docs/ + trim_generic docs + ;; + bzip2) + rm -f manual.* sample* ./*.html + trim_generic + ;; + lz4) + # Only lib/ is built (LZ4_BUILD_CLI=OFF). Files outside of lib/ are + # GPL-2.0-or-later, so don't ship the ones we don't need. + rmd programs contrib ossfuzz + trim_generic + (cd build && keep_only cmake) + ;; + mimalloc) + rmd ide + trim_generic + ;; + rapidjson) + rmd bin thirdparty + trim_generic + ;; + re2) + rmd benchlog python app + trim_generic + ;; + simdjson) + # Built from src/ and include/. singleheader/ is a 20 MB amalgamation. + rmd singleheader images jsonexamples tools scripts + trim_generic + ;; + thrift) + # Only the C++ library is built (BUILD_COMPILER=OFF, WITH_*=OFF) + rmd tutorial test compiler contrib doc debian + (cd lib && keep_only cpp CMakeLists.txt) + trim_generic + ;; + zlib) + rmd contrib + # zlib's CMakeLists.txt builds test/example.c unless ZLIB_BUILD_EXAMPLES=OFF + trim_generic test + ;; + zstd) + # ZSTD_BUILD_PROGRAMS=OFF + rmd contrib zlibWrapper programs + trim_generic + (cd build && keep_only cmake) + ;; + *) + trim_generic + ;; + esac +} + +# Extract $1, trim it for dependency $3 and write the result to $2 +trim_tarball() { + local in="$1" out="$2" name="$3" + local work src + if [ -f "${SOURCE_DIR}/trim/${name}.sh" ]; then + bash "${SOURCE_DIR}/trim/${name}.sh" "${in}" "${out}" + return + fi + work="$(mktemp -d)" + # Use CMake's tar (libarchive) for .tar.bz2: tar -j needs a bzip2 binary, + # which e.g. the R CI images don't have. + case "${in}" in + *.tar.bz2) (cd "${work}" && cmake -E tar xf "${in}") ;; + *) tar -xf "${in}" -C "${work}" ;; + esac + # Every tarball we vendor has a single top-level directory + src="$(cd "${work}" && ls)" + if [ "$(echo "${src}" | wc -l)" -ne 1 ] || [ ! -d "${work}/${src}" ]; then + echo "Expected a single top-level directory in ${in}" >&2 + rm -rf "${work}" + return 1 + fi + (cd "${work}/${src}" && trim_dependency "${name}") + case "${out}" in + *.tar.bz2) (cd "${work}" && cmake -E tar cjf "${out}" "${src}") ;; + *.tar.gz) tar -czf "${out}" -C "${work}" "${src}" ;; + *) + echo "Don't know how to compress ${out}" >&2 + rm -rf "${work}" + return 1 + ;; + esac + rm -rf "${work}" +} + +mkdir -p "${DESTDIR}" "${DOWNLOAD_DIR}" +# trim_tarball() changes directories, so it needs absolute paths +DOWNLOAD_DIR="$(cd "${DOWNLOAD_DIR}" && pwd)" +TRIM_DIR="$(mktemp -d)" +trap 'rm -rf "${TRIM_DIR}"' EXIT +keep=(checksums.txt) +sums="filename upstream_sha256 vendored_sha256"$'\n' for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do url="" for dep_packed in "${DEPENDENCIES[@]}"; do @@ -67,28 +259,38 @@ for vendored_url_var in "${VENDORED_DEPENDENCIES[@]}"; do echo "${vendored_url_var} not found in versions.txt" >&2 exit 1 fi - # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM + # e.g. ARROW_XSIMD_URL -> ARROW_XSIMD_BUILD_SHA256_CHECKSUM and xsimd checksum_var="${vendored_url_var%_URL}_BUILD_SHA256_CHECKSUM" checksum="${!checksum_var}" + name="$(echo "${vendored_url_var}" | sed -e 's/^ARROW_//' -e 's/_URL$//' | tr '[:upper:]' '[:lower:]')" keep+=("${tar_name}") - out="${DESTDIR}/${tar_name}" - if [ -f "${out}" ] && [ "$(sha256 "${out}")" = "${checksum}" ]; then - continue + upstream="${DOWNLOAD_DIR}/${tar_name}" + if [ ! -f "${upstream}" ] || [ "$(sha256 "${upstream}")" != "${checksum}" ]; then + echo "Downloading ${url}" + curl --fail --silent --show-error --location --retry 3 --output "${upstream}" "${url}" + if [ "$(sha256 "${upstream}")" != "${checksum}" ]; then + echo "Checksum mismatch for ${upstream}" >&2 + rm -f "${upstream}" + exit 1 + fi fi - echo "Downloading ${url}" - curl --fail --silent --show-error --location --retry 3 --output "${out}" "${url}" - - if [ "$(sha256 "${out}")" != "${checksum}" ]; then - echo "Checksum mismatch for ${out}" >&2 - rm -f "${out}" - exit 1 - fi + # Trim into a temporary directory (keeping the file name, so the extension + # says how to compress it) so that a failure doesn't leave a partial file. + echo "Trimming ${tar_name}" + trim_tarball "${upstream}" "${TRIM_DIR}/${tar_name}" "${name}" + out="${DESTDIR}/${tar_name}" + mv "${TRIM_DIR}/${tar_name}" "${out}" + sums="${sums}${tar_name} ${checksum} $(sha256 "${out}")"$'\n' done -# Remove tarballs for other versions +# See read_vendored_checksums() in nixlibs.R +printf "%s" "${sums}" > "${DESTDIR}/checksums.txt" + +# Remove tarballs for other versions (or dependencies we no longer vendor) for file in "${DESTDIR}"/*; do + [ -e "${file}" ] || continue stale=true for name in "${keep[@]}"; do if [ "$(basename "${file}")" = "${name}" ]; then