diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index 8931bfbfa9e0..a64ca50b61e8 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -27,6 +27,9 @@ function check_arrow_visibility { if [[ "${ARROW_GCS}" == "ON" ]]; then libs+=(libarrow_gcs.so) fi + if [[ "${ARROW_AZURE}" == "ON" ]]; then + libs+=(libarrow_azure.so) + fi for lib in "${libs[@]}"; do nm --demangle --dynamic "/tmp/arrow-dist/lib/${lib}" done > nm_arrow.log diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 123def5b66c5..62f562e7d52c 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -4440,13 +4440,19 @@ function(build_azure_sdk) endfunction() if(ARROW_WITH_AZURE_SDK) - resolve_dependency(Azure REQUIRED_VERSION 1.10.2) + resolve_dependency(Azure + ARROW_CMAKE_PACKAGE_NAME + ArrowAzure + ARROW_PC_PACKAGE_NAME + arrow-azure + REQUIRED_VERSION + 1.10.2) set(AZURE_SDK_LINK_LIBRARIES Azure::azure-storage-files-datalake Azure::azure-storage-blobs Azure::azure-identity) if(AZURE_SDK_VENDORED AND NOT WIN32) - find_curl(ARROW) + find_curl(ARROW_AZURE) find_package(LibXml2 REQUIRED) - list(APPEND ARROW_SYSTEM_DEPENDENCIES LibXml2) + list(APPEND ARROW_AZURE_SYSTEM_DEPENDENCIES LibXml2) endif() endif() @@ -4457,12 +4463,13 @@ if(ARROW_BUILD_STATIC) if(ARROW_AZURE AND AZURE_SDK_VENDORED AND NOT WIN32) - arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl" ARROW) + arrow_append_pc_system_library("libcurl" ARROW_AZURE_CURL_PC "-lcurl" ARROW_AZURE) endif() if(ARROW_AZURE AND AZURE_SDK_VENDORED AND NOT WIN32) - arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2" ARROW) + arrow_append_pc_system_library("libxml-2.0" ARROW_AZURE_LIBXML2_PC "-lxml2" + ARROW_AZURE) endif() if(ARROW_GCS AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED" diff --git a/cpp/src/arrow/ArrowAzureConfig.cmake.in b/cpp/src/arrow/ArrowAzureConfig.cmake.in new file mode 100644 index 000000000000..c6257ee70ad9 --- /dev/null +++ b/cpp/src/arrow/ArrowAzureConfig.cmake.in @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# This config sets the following variables in your project:: +# +# ArrowAzure_FOUND - true if Arrow Azure found on the system +# +# This config sets the following targets in your project:: +# +# ArrowAzure::arrow_azure_shared - for linked as shared library if shared library is built +# ArrowAzure::arrow_azure_static - for linked as static library if static library is built + +@PACKAGE_INIT@ + +set(ARROW_AZURE_SYSTEM_DEPENDENCIES "@ARROW_AZURE_SYSTEM_DEPENDENCIES@") + +include(CMakeFindDependencyMacro) +find_dependency(Arrow CONFIG) + +if(ARROW_BUILD_STATIC) + arrow_find_dependencies("${ARROW_AZURE_SYSTEM_DEPENDENCIES}") +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/ArrowAzureTargets.cmake") + +arrow_keep_backward_compatibility(ArrowAzure arrow_azure) + +check_required_components(ArrowAzure) + +arrow_show_details(ArrowAzure ARROW_AZURE) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index e71d92fdb880..fc04c45e601f 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -33,16 +33,6 @@ set(ARROW_SHARED_PRIVATE_LINK_LIBS) set(ARROW_SHARED_INSTALL_INTERFACE_LIBS) set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) -if(ARROW_AZURE) - if(Azure_SOURCE STREQUAL "SYSTEM") - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES}) - elseif(Azure_SOURCE STREQUAL "BUNDLED") - if(NOT WIN32) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl LibXml2::LibXml2) - endif() - endif() -endif() - if(ARROW_USE_OPENSSL) list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) endif() @@ -964,9 +954,61 @@ if(ARROW_FILESYSTEM) filesystem/util_internal.cc) if(ARROW_AZURE) - list(APPEND ARROW_FILESYSTEM_SRCS filesystem/azurefs.cc) - set_source_files_properties(filesystem/azurefs.cc + set(ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static) + if(Azure_SOURCE STREQUAL "SYSTEM") + list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES}) + elseif(Azure_SOURCE STREQUAL "BUNDLED") + if(NOT WIN32) + list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl + LibXml2::LibXml2) + endif() + endif() + set(ARROW_AZURE_LIB_SRCS filesystem/azurefs_module.cc filesystem/azurefs.cc) + set_source_files_properties(filesystem/azurefs_module.cc filesystem/azurefs.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + # If libarrow_azure.a is only built, "pkg-config --cflags --libs + # arrow-azure" outputs build flags for static linking not shared + # linking. ARROW_AZURE_PC_* except ARROW_AZURE_PC_*_PRIVATE are for the + # static linking case. + if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC) + string(APPEND ARROW_AZURE_PC_CFLAGS "${ARROW_AZURE_PC_CFLAGS_PRIVATE}") + set(ARROW_AZURE_PC_CFLAGS_PRIVATE "") + set(ARROW_AZURE_PC_LIBS "${ARROW_AZURE_PC_LIBS_PRIVATE}") + set(ARROW_AZURE_PC_LIBS_PRIVATE "") + set(ARROW_AZURE_PC_REQUIRES "${ARROW_AZURE_PC_REQUIRES_PRIVATE}") + set(ARROW_AZURE_PC_REQUIRES_PRIVATE "") + else() + set(ARROW_AZURE_PC_LIBS "") + set(ARROW_AZURE_PC_REQUIRES "") + endif() + add_arrow_lib(arrow_azure + CMAKE_PACKAGE_NAME + ArrowAzure + PKG_CONFIG_NAME + arrow-azure + SOURCES + ${ARROW_AZURE_LIB_SRCS} + SHARED_LINK_LIBS + arrow_shared + SHARED_PRIVATE_LINK_LIBS + ${AZURE_SDK_LINK_LIBRARIES} + SHARED_INSTALL_INTERFACE_LIBS + Arrow::arrow_shared + STATIC_LINK_LIBS + arrow_static + ${AZURE_SDK_LINK_LIBRARIES} + STATIC_INSTALL_INTERFACE_LIBS + ${ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS} + SHARED_LINK_FLAGS + ${ARROW_VERSION_SCRIPT_FLAGS} + OUTPUTS + ARROW_AZURE_LIBRARIES) + foreach(LIB_TARGET ${ARROW_AZURE_LIBRARIES}) + target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_AZURE_EXPORTING) + endforeach() + if(ARROW_BUILD_STATIC AND WIN32) + target_compile_definitions(arrow_azure_static PUBLIC ARROW_AZURE_STATIC) + endif() endif() if(ARROW_GCS) set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static) @@ -1058,12 +1100,6 @@ if(ARROW_FILESYSTEM) endif() arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS}) - if(ARROW_AZURE) - foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS}) - target_link_libraries(${ARROW_FILESYSTEM_TARGET} - PRIVATE ${AZURE_SDK_LINK_LIBRARIES}) - endforeach() - endif() if(ARROW_HDFS) foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS}) target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE arrow::hadoop) diff --git a/cpp/src/arrow/arrow-azure.pc.in b/cpp/src/arrow/arrow-azure.pc.in new file mode 100644 index 000000000000..dca6f722e4f6 --- /dev/null +++ b/cpp/src/arrow/arrow-azure.pc.in @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +prefix=@CMAKE_INSTALL_PREFIX@ +includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@ +libdir=@ARROW_PKG_CONFIG_LIBDIR@ + +Name: Apache Arrow Azure +Description: Apache Arrow's Azure filesystem implementation. +Version: @ARROW_VERSION@ +Requires: arrow@ARROW_AZURE_PC_REQUIRES@ +Requires.private:@ARROW_AZURE_PC_REQUIRES_PRIVATE@ +Libs: -L${libdir} -larrow_azure@ARROW_AZURE_PC_LIBS@ +Libs.private:@ARROW_AZURE_PC_LIBS_PRIVATE@ +Cflags:@ARROW_AZURE_PC_CFLAGS@ +Cflags.private:@ARROW_AZURE_PC_CFLAGS_PRIVATE@ diff --git a/cpp/src/arrow/filesystem/CMakeLists.txt b/cpp/src/arrow/filesystem/CMakeLists.txt index 154c2161938a..9c2772a99d36 100644 --- a/cpp/src/arrow/filesystem/CMakeLists.txt +++ b/cpp/src/arrow/filesystem/CMakeLists.txt @@ -75,11 +75,29 @@ if(ARROW_GCS) endif() if(ARROW_AZURE) + set(ARROW_AZURE_TEST_EXTRA_LINK_LIBS) + if(ARROW_TEST_LINKAGE STREQUAL "shared") + list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS arrow_azure_shared) + else() + list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS arrow_azure_static) + endif() + list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS ${AZURE_SDK_LINK_LIBRARIES}) add_arrow_test(azurefs_test EXTRA_LABELS filesystem EXTRA_LINK_LIBS - ${AZURE_SDK_LINK_LIBRARIES}) + ${ARROW_AZURE_TEST_EXTRA_LINK_LIBS}) + if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED) + add_arrow_test(azurefs_module_test + SOURCES + azurefs_module_test.cc + EXTRA_LABELS + filesystem + DEFINITIONS + ARROW_AZURE_LIBPATH="$" + EXTRA_DEPENDENCIES + arrow_azure_shared) + endif() endif() if(ARROW_S3) diff --git a/cpp/src/arrow/filesystem/azure_visibility.h b/cpp/src/arrow/filesystem/azure_visibility.h new file mode 100644 index 000000000000..aa62b5ba53c1 --- /dev/null +++ b/cpp/src/arrow/filesystem/azure_visibility.h @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#if defined(_WIN32) || defined(__CYGWIN__) +# if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4251) +# else +# pragma GCC diagnostic ignored "-Wattributes" +# endif + +# ifdef ARROW_AZURE_STATIC +# define ARROW_AZURE_EXPORT +# elif defined(ARROW_AZURE_EXPORTING) +# define ARROW_AZURE_EXPORT __declspec(dllexport) +# else +# define ARROW_AZURE_EXPORT __declspec(dllimport) +# endif + +# define ARROW_AZURE_NO_EXPORT + +# if defined(_MSC_VER) +# pragma warning(pop) +# endif + +#else // Not Windows +# ifndef ARROW_AZURE_EXPORT +# define ARROW_AZURE_EXPORT __attribute__((visibility("default"))) +# endif +# ifndef ARROW_AZURE_NO_EXPORT +# define ARROW_AZURE_NO_EXPORT __attribute__((visibility("hidden"))) +# endif +#endif diff --git a/cpp/src/arrow/filesystem/azurefs.cc b/cpp/src/arrow/filesystem/azurefs.cc index 20b0a655d8e2..7e97d74b4ccb 100644 --- a/cpp/src/arrow/filesystem/azurefs.cc +++ b/cpp/src/arrow/filesystem/azurefs.cc @@ -3561,4 +3561,15 @@ Result AzureFileSystem::PathFromUri(const std::string& uri_string) return path; } +static Result> AzureFileSystemFactory( + const Uri& uri, const io::IOContext& io_context, std::string* out_path) { + ARROW_ASSIGN_OR_RAISE(auto options, AzureOptions::FromUri(uri, out_path)); + return AzureFileSystem::Make(options, io_context); +} + +FileSystemRegistrar kAzureFileSystemRegistrar[]{ + ARROW_REGISTER_FILESYSTEM("abfs", AzureFileSystemFactory, {}), + ARROW_REGISTER_FILESYSTEM("abfss", AzureFileSystemFactory, {}), +}; + } // namespace arrow::fs diff --git a/cpp/src/arrow/filesystem/azurefs.h b/cpp/src/arrow/filesystem/azurefs.h index d21bcd08df81..dc70d2b06c34 100644 --- a/cpp/src/arrow/filesystem/azurefs.h +++ b/cpp/src/arrow/filesystem/azurefs.h @@ -21,6 +21,7 @@ #include #include +#include "arrow/filesystem/azure_visibility.h" #include "arrow/filesystem/filesystem.h" #include "arrow/util/macros.h" #include "arrow/util/uri.h" @@ -59,7 +60,7 @@ class TestAzureOptions; /// - `AZURE_FEDERATED_TOKEN_FILE` /// /// Functions are provided for explicit configuration of credentials if that is preferred. -struct ARROW_EXPORT AzureOptions { +struct ARROW_AZURE_EXPORT AzureOptions { friend class TestAzureOptions; /// \brief The name of the Azure Storage Account being accessed. @@ -247,7 +248,7 @@ struct ARROW_EXPORT AzureOptions { /// [2]: https://azure.microsoft.com/en-us/products/storage/data-lake-storage /// [3]: /// https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-known-issues -class ARROW_EXPORT AzureFileSystem : public FileSystem { +class ARROW_AZURE_EXPORT AzureFileSystem : public FileSystem { private: class Impl; std::unique_ptr impl_; diff --git a/cpp/src/arrow/filesystem/azurefs_internal.h b/cpp/src/arrow/filesystem/azurefs_internal.h index c6d730fcb938..836ce825ecb3 100644 --- a/cpp/src/arrow/filesystem/azurefs_internal.h +++ b/cpp/src/arrow/filesystem/azurefs_internal.h @@ -17,6 +17,7 @@ #pragma once +#include "arrow/filesystem/azure_visibility.h" #include "arrow/result.h" namespace Azure::Storage::Files::DataLake { @@ -70,7 +71,8 @@ enum class HierarchicalNamespaceSupport { /// account. /// \return kEnabled/kDisabled/kContainerNotFound (kUnknown is never /// returned). -ARROW_EXPORT Result CheckIfHierarchicalNamespaceIsEnabled( +ARROW_AZURE_EXPORT Result +CheckIfHierarchicalNamespaceIsEnabled( const Azure::Storage::Files::DataLake::DataLakeFileSystemClient& adlfs_client, const arrow::fs::AzureOptions& options); diff --git a/cpp/src/arrow/filesystem/azurefs_module.cc b/cpp/src/arrow/filesystem/azurefs_module.cc new file mode 100644 index 000000000000..d91413138b45 --- /dev/null +++ b/cpp/src/arrow/filesystem/azurefs_module.cc @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/filesystem/filesystem_library.h" diff --git a/cpp/src/arrow/filesystem/azurefs_module_test.cc b/cpp/src/arrow/filesystem/azurefs_module_test.cc new file mode 100644 index 000000000000..7317385dd5d5 --- /dev/null +++ b/cpp/src/arrow/filesystem/azurefs_module_test.cc @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "arrow/filesystem/filesystem.h" +#include "arrow/testing/gtest_util.h" + +namespace arrow::fs { + +class RegistrationTestEnvironment : public ::testing::Environment { + public: + void SetUp() override { ASSERT_OK(LoadFileSystemFactories(ARROW_AZURE_LIBPATH)); } +}; + +auto* lib_env = ::testing::AddGlobalTestEnvironment(new RegistrationTestEnvironment); + +TEST(AzureModuleTest, FromUri) { + std::string path; + for (const auto* scheme : {"abfs", "abfss"}) { + ASSERT_OK_AND_ASSIGN( + auto fs, FileSystemFromUri(std::string(scheme) + + "://account.blob.core.windows.net/container/dir/" + "blob?credential_kind=anonymous", + &path)); + EXPECT_EQ(fs->type_name(), "abfs"); + EXPECT_EQ(path, "container/dir/blob"); + } +} + +TEST(AzureModuleTest, FromUriAndOptionsNotImplemented) { + EXPECT_RAISES_WITH_MESSAGE_THAT( + NotImplemented, ::testing::HasSubstr("Filesystem factory does not support"), + FileSystemFromUriAndOptions( + "abfs://account.blob.core.windows.net/container?credential_kind=anonymous", + {{"foo", std::string{"bar"}}})); +} + +} // namespace arrow::fs diff --git a/cpp/src/arrow/filesystem/azurefs_test.cc b/cpp/src/arrow/filesystem/azurefs_test.cc index 250e3572bd9b..f5adbb8e1f2d 100644 --- a/cpp/src/arrow/filesystem/azurefs_test.cc +++ b/cpp/src/arrow/filesystem/azurefs_test.cc @@ -509,6 +509,24 @@ TEST(AzureFileSystem, InitializingFilesystemWithoutAccountNameFails) { ASSERT_RAISES(Invalid, AzureFileSystem::Make(options)); } +TEST(AzureFileSystem, FileSystemFromUriRegistry) { + std::string path; + for (const auto& scheme : {"abfs", "abfss"}) { + ASSERT_OK_AND_ASSIGN( + auto fs, FileSystemFromUri(std::string(scheme) + + "://account.blob.core.windows.net/container/dir/" + "blob?credential_kind=anonymous", + &path)); + EXPECT_EQ(fs->type_name(), "abfs"); + EXPECT_EQ(path, "container/dir/blob"); + } + EXPECT_RAISES_WITH_MESSAGE_THAT( + NotImplemented, ::testing::HasSubstr("Filesystem factory does not support"), + FileSystemFromUriAndOptions( + "abfs://account.blob.core.windows.net/container?credential_kind=anonymous", + {{"foo", std::string{"bar"}}})); +} + TEST(AzureFileSystem, InitializeWithDefaultCredential) { AzureOptions options; options.account_name = "dummy-account-name"; diff --git a/cpp/src/arrow/filesystem/filesystem.cc b/cpp/src/arrow/filesystem/filesystem.cc index c15bc06681d6..6b49f05eb0b1 100644 --- a/cpp/src/arrow/filesystem/filesystem.cc +++ b/cpp/src/arrow/filesystem/filesystem.cc @@ -25,9 +25,6 @@ #include "arrow/util/config.h" #include "arrow/filesystem/filesystem.h" -#ifdef ARROW_AZURE -# include "arrow/filesystem/azurefs.h" -#endif #ifdef ARROW_HDFS # include "arrow/filesystem/hdfs.h" #endif @@ -909,16 +906,6 @@ Result> FileSystemFromUriReal( scheme, "', got ", options.size(), " option(s)"); } - if (scheme == "abfs" || scheme == "abfss") { -#ifdef ARROW_AZURE - ARROW_ASSIGN_OR_RAISE(auto options, AzureOptions::FromUri(uri, out_path)); - return AzureFileSystem::Make(options, io_context); -#else - return Status::NotImplemented( - "Got Azure Blob File System URI but Arrow compiled without Azure Blob File " - "System support"); -#endif - } if (scheme == "hdfs" || scheme == "viewfs") { #ifdef ARROW_HDFS ARROW_ASSIGN_OR_RAISE(auto options, HdfsOptions::FromUri(uri)); diff --git a/cpp/src/arrow/filesystem/meson.build b/cpp/src/arrow/filesystem/meson.build index e2eaa116a710..91183e257897 100644 --- a/cpp/src/arrow/filesystem/meson.build +++ b/cpp/src/arrow/filesystem/meson.build @@ -18,6 +18,7 @@ install_headers( [ 'api.h', + 'azure_visibility.h', 'azurefs.h', 'filesystem.h', 'filesystem_library.h', diff --git a/dev/release/verify-apt.sh b/dev/release/verify-apt.sh index 38051c9a43ec..ced705937f63 100755 --- a/dev/release/verify-apt.sh +++ b/dev/release/verify-apt.sh @@ -299,6 +299,7 @@ ${APT_INSTALL} --allow-downgrades \ libarrow-dev=${previous_package_version} \ libarrow-glib${previous_so_version}=${previous_package_version} \ libarrow-glib-dev=${previous_package_version} \ + libarrow-s3-${previous_so_version}=${previous_package_version} \ libgandiva${previous_so_version}=${previous_package_version} \ libgandiva-dev=${previous_package_version} \ libgandiva-glib${previous_so_version}=${previous_package_version} \ @@ -327,7 +328,8 @@ ${APT_INSTALL} --allow-downgrades \ libarrow-flight-sql-glib${previous_so_version}=${previous_package_version} \ libarrow-flight-sql-glib-dev=${previous_package_version} \ libarrow-glib${previous_so_version}=${previous_package_version} \ - libarrow-glib-dev=${previous_package_version} + libarrow-glib-dev=${previous_package_version} \ + libarrow-s3-${previous_so_version}=${previous_package_version} echo "::endgroup::" echo "::group::Downgrade Apache Arrow Dataset" @@ -347,6 +349,7 @@ ${APT_INSTALL} --allow-downgrades \ libarrow-dev=${previous_package_version} \ libarrow-glib${previous_so_version}=${previous_package_version} \ libarrow-glib-dev=${previous_package_version} \ + libarrow-s3-${previous_so_version}=${previous_package_version} \ libparquet${previous_so_version}=${previous_package_version} \ libparquet-dev=${previous_package_version} \ libparquet-glib${previous_so_version}=${previous_package_version} \ diff --git a/dev/tasks/linux-packages/apache-arrow/debian/control.in b/dev/tasks/linux-packages/apache-arrow/debian/control.in index 5609724a56df..a945973a89b8 100644 --- a/dev/tasks/linux-packages/apache-arrow/debian/control.in +++ b/dev/tasks/linux-packages/apache-arrow/debian/control.in @@ -155,6 +155,19 @@ Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for GCS filesystem module. +Package: libarrow-azure-2700 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow2700 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for Azure filesystem module. + Package: libarrow-flight2700 Section: libs Architecture: any @@ -289,6 +302,18 @@ Description: Apache Arrow is a data processing library for analysis . This package provides C++ library files for GCS filesystem module. +Package: libarrow-azure-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-azure-2700 (= ${binary:Version}), + libarrow-dev (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for Azure filesystem module. + Package: libarrow-flight-dev Section: libdevel Architecture: any diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-2700.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-2700.install new file mode 100644 index 000000000000..fcc16570a402 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-2700.install @@ -0,0 +1 @@ +usr/lib/*/libarrow_azure.so.* diff --git a/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install new file mode 100644 index 000000000000..3209f6d9ed85 --- /dev/null +++ b/dev/tasks/linux-packages/apache-arrow/debian/libarrow-azure-dev.install @@ -0,0 +1,4 @@ +usr/lib/*/cmake/ArrowAzure/ +usr/lib/*/libarrow_azure.a +usr/lib/*/libarrow_azure.so +usr/lib/*/pkgconfig/arrow-azure.pc diff --git a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in index c4a10036ecfe..757a9e4b648c 100644 --- a/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in +++ b/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -392,6 +392,38 @@ Libraries and header files for Apache Arrow GCS filesystem. %{_libdir}/pkgconfig/arrow-gcs.pc %endif +%package -n %{name}%{so_version}-azure-libs +Summary: C++ library for the Azure filesystem +License: Apache-2.0 +Requires: %{name}%{so_version}-libs = %{version}-%{release} + +%description -n %{name}%{so_version}-azure-libs +This package contains the libraries for Apache Arrow Azure filesystem. + +%files -n %{name}%{so_version}-azure-libs +%defattr(-,root,root,-) +%doc README.md +%license LICENSE.txt NOTICE.txt +%{_libdir}/libarrow_azure.so.* + +%package azure-devel +Summary: Libraries and header files for Apache Arrow Azure filesystem +License: Apache-2.0 +Requires: %{name}%{so_version}-azure-libs = %{version}-%{release} +Requires: %{name}-devel = %{version}-%{release} + +%description azure-devel +Libraries and header files for Apache Arrow Azure filesystem. + +%files azure-devel +%defattr(-,root,root,-) +%doc README.md +%license LICENSE.txt NOTICE.txt +%{_libdir}/cmake/ArrowAzure/ +%{_libdir}/libarrow_azure.a +%{_libdir}/libarrow_azure.so +%{_libdir}/pkgconfig/arrow-azure.pc + %package -n %{name}%{so_version}-acero-libs Summary: C++ library to execute a query in streaming License: Apache-2.0 diff --git a/docs/source/cpp/build_system.rst b/docs/source/cpp/build_system.rst index 800631b6e098..6a61f5152d6d 100644 --- a/docs/source/cpp/build_system.rst +++ b/docs/source/cpp/build_system.rst @@ -115,6 +115,7 @@ This is the list of available packages: * ArrowCUDA * ArrowAcero +* ArrowAzure * ArrowCompute * ArrowDataset * ArrowFlight @@ -186,6 +187,7 @@ Available packages The Arrow C++ provides a pkg-config package for each module. Here are all available packages: + * ``arrow-azure`` * ``arrow-csv`` * ``arrow-cuda`` * ``arrow-dataset`` diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 5bff8bcc6499..f1f35ed365bb 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -728,6 +728,18 @@ if(PYARROW_BUILD_AZURE) if(NOT ARROW_AZURE) message(FATAL_ERROR "You must build Arrow C++ with ARROW_AZURE=ON") endif() + find_package(ArrowAzure REQUIRED) + if(ARROW_BUILD_SHARED) + if(PYARROW_BUNDLE_ARROW_CPP) + bundle_arrow_lib(${ARROW_AZURE_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION}) + if(MSVC) + bundle_arrow_import_lib(${ARROW_AZURE_IMPORT_LIB}) + endif() + endif() + set(AZURE_LINK_LIBS ArrowAzure::arrow_azure_shared) + else() + set(AZURE_LINK_LIBS ArrowAzure::arrow_azure_static) + endif() list(APPEND CYTHON_EXTENSIONS _azurefs) endif() @@ -1064,6 +1076,10 @@ if(PYARROW_BUILD_PARQUET) endif() endif() +if(PYARROW_BUILD_AZURE) + target_link_libraries(_azurefs PRIVATE ${AZURE_LINK_LIBS}) +endif() + if(PYARROW_BUILD_GCS) target_link_libraries(_gcsfs PRIVATE ${GCS_LINK_LIBS}) endif() diff --git a/r/configure b/r/configure index 87b48d572887..b19ebbec543f 100755 --- a/r/configure +++ b/r/configure @@ -365,6 +365,10 @@ add_feature_flags () { fi if arrow_built_with ARROW_AZURE; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" + PKG_CONFIG_NAMES_FEATURES="$PKG_CONFIG_NAMES_FEATURES arrow-azure" + PKG_LIBS_FEATURES_WITHOUT_PC="-larrow_azure $PKG_LIBS_FEATURES_WITHOUT_PC" + # NOTE: arrow_azure is assumed to have the same -L flag as arrow + # so there is no need to add its location to PKG_DIRS PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lxml2" fi if arrow_built_with ARROW_GCS; then