GH-55258: [C++] Move Azure and its SDK to its own libarrow_azure.so

ci/scripts/python_wheel_xlinux_build.sh

@@ -27,6 +27,9 @@ function check_arrow_visibility {
     if [[ "${ARROW_GCS}" == "ON" ]]; then
         libs+=(libarrow_gcs.so)
     fi
+    if [[ "${ARROW_AZURE}" == "ON" ]]; then
+        libs+=(libarrow_azure.so)
+    fi
     for lib in "${libs[@]}"; do
         nm --demangle --dynamic "/tmp/arrow-dist/lib/${lib}"
     done > nm_arrow.log

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -4440,13 +4440,19 @@ function(build_azure_sdk)
 endfunction()
 
 if(ARROW_WITH_AZURE_SDK)
-  resolve_dependency(Azure REQUIRED_VERSION 1.10.2)
+  resolve_dependency(Azure
+                     ARROW_CMAKE_PACKAGE_NAME
+                     ArrowAzure
+                     ARROW_PC_PACKAGE_NAME
+                     arrow-azure
+                     REQUIRED_VERSION
+                     1.10.2)
   set(AZURE_SDK_LINK_LIBRARIES Azure::azure-storage-files-datalake
                                Azure::azure-storage-blobs Azure::azure-identity)
   if(AZURE_SDK_VENDORED AND NOT WIN32)
-    find_curl(ARROW)
+    find_curl(ARROW_AZURE)
     find_package(LibXml2 REQUIRED)
-    list(APPEND ARROW_SYSTEM_DEPENDENCIES LibXml2)
+    list(APPEND ARROW_AZURE_SYSTEM_DEPENDENCIES LibXml2)
   endif()
 endif()
 
@@ -4457,12 +4463,13 @@ if(ARROW_BUILD_STATIC)
   if(ARROW_AZURE
      AND AZURE_SDK_VENDORED
      AND NOT WIN32)
-    arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl" ARROW)
+    arrow_append_pc_system_library("libcurl" ARROW_AZURE_CURL_PC "-lcurl" ARROW_AZURE)
   endif()
   if(ARROW_AZURE
      AND AZURE_SDK_VENDORED
      AND NOT WIN32)
-    arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2" ARROW)
+    arrow_append_pc_system_library("libxml-2.0" ARROW_AZURE_LIBXML2_PC "-lxml2"
+                                   ARROW_AZURE)
   endif()
   if(ARROW_GCS
      AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED"

cpp/src/arrow/ArrowAzureConfig.cmake.in

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This config sets the following variables in your project::
+#
+#   ArrowAzure_FOUND - true if Arrow Azure found on the system
+#
+# This config sets the following targets in your project::
+#
+#   ArrowAzure::arrow_azure_shared - for linked as shared library if shared library is built
+#   ArrowAzure::arrow_azure_static - for linked as static library if static library is built
+
+@PACKAGE_INIT@
+
+set(ARROW_AZURE_SYSTEM_DEPENDENCIES "@ARROW_AZURE_SYSTEM_DEPENDENCIES@")
+
+include(CMakeFindDependencyMacro)
+find_dependency(Arrow CONFIG)
+
+if(ARROW_BUILD_STATIC)
+  arrow_find_dependencies("${ARROW_AZURE_SYSTEM_DEPENDENCIES}")
+endif()
+
+include("${CMAKE_CURRENT_LIST_DIR}/ArrowAzureTargets.cmake")
+
+arrow_keep_backward_compatibility(ArrowAzure arrow_azure)
+
+check_required_components(ArrowAzure)
+
+arrow_show_details(ArrowAzure ARROW_AZURE)

cpp/src/arrow/CMakeLists.txt

@@ -33,16 +33,6 @@ set(ARROW_SHARED_PRIVATE_LINK_LIBS)
 set(ARROW_SHARED_INSTALL_INTERFACE_LIBS)
 set(ARROW_STATIC_INSTALL_INTERFACE_LIBS)
 
-if(ARROW_AZURE)
-  if(Azure_SOURCE STREQUAL "SYSTEM")
-    list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES})
-  elseif(Azure_SOURCE STREQUAL "BUNDLED")
-    if(NOT WIN32)
-      list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl LibXml2::LibXml2)
-    endif()
-  endif()
-endif()
-
 if(ARROW_USE_OPENSSL)
   list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS})
 endif()
@@ -964,9 +954,61 @@ if(ARROW_FILESYSTEM)
       filesystem/util_internal.cc)
 
   if(ARROW_AZURE)
-    list(APPEND ARROW_FILESYSTEM_SRCS filesystem/azurefs.cc)
-    set_source_files_properties(filesystem/azurefs.cc
+    set(ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
+    if(Azure_SOURCE STREQUAL "SYSTEM")
+      list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES})
+    elseif(Azure_SOURCE STREQUAL "BUNDLED")
+      if(NOT WIN32)
+        list(APPEND ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl
+             LibXml2::LibXml2)
+      endif()
+    endif()
+    set(ARROW_AZURE_LIB_SRCS filesystem/azurefs_module.cc filesystem/azurefs.cc)
+    set_source_files_properties(filesystem/azurefs_module.cc filesystem/azurefs.cc
                                 PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
+    # If libarrow_azure.a is only built, "pkg-config --cflags --libs
+    # arrow-azure" outputs build flags for static linking not shared
+    # linking. ARROW_AZURE_PC_* except ARROW_AZURE_PC_*_PRIVATE are for the
+    # static linking case.
+    if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
+      string(APPEND ARROW_AZURE_PC_CFLAGS "${ARROW_AZURE_PC_CFLAGS_PRIVATE}")
+      set(ARROW_AZURE_PC_CFLAGS_PRIVATE "")
+      set(ARROW_AZURE_PC_LIBS "${ARROW_AZURE_PC_LIBS_PRIVATE}")
+      set(ARROW_AZURE_PC_LIBS_PRIVATE "")
+      set(ARROW_AZURE_PC_REQUIRES "${ARROW_AZURE_PC_REQUIRES_PRIVATE}")
+      set(ARROW_AZURE_PC_REQUIRES_PRIVATE "")
+    else()
+      set(ARROW_AZURE_PC_LIBS "")
+      set(ARROW_AZURE_PC_REQUIRES "")
+    endif()
+    add_arrow_lib(arrow_azure
+                  CMAKE_PACKAGE_NAME
+                  ArrowAzure
+                  PKG_CONFIG_NAME
+                  arrow-azure
+                  SOURCES
+                  ${ARROW_AZURE_LIB_SRCS}
+                  SHARED_LINK_LIBS
+                  arrow_shared
+                  SHARED_PRIVATE_LINK_LIBS
+                  ${AZURE_SDK_LINK_LIBRARIES}
+                  SHARED_INSTALL_INTERFACE_LIBS
+                  Arrow::arrow_shared
+                  STATIC_LINK_LIBS
+                  arrow_static
+                  ${AZURE_SDK_LINK_LIBRARIES}
+                  STATIC_INSTALL_INTERFACE_LIBS
+                  ${ARROW_AZURE_STATIC_INSTALL_INTERFACE_LIBS}
+                  SHARED_LINK_FLAGS
+                  ${ARROW_VERSION_SCRIPT_FLAGS}
+                  OUTPUTS
+                  ARROW_AZURE_LIBRARIES)
+    foreach(LIB_TARGET ${ARROW_AZURE_LIBRARIES})
+      target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_AZURE_EXPORTING)
+    endforeach()
+    if(ARROW_BUILD_STATIC AND WIN32)
+      target_compile_definitions(arrow_azure_static PUBLIC ARROW_AZURE_STATIC)
+    endif()
   endif()
   if(ARROW_GCS)
     set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
@@ -1058,12 +1100,6 @@ if(ARROW_FILESYSTEM)
   endif()
 
   arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS})
-  if(ARROW_AZURE)
-    foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
-      target_link_libraries(${ARROW_FILESYSTEM_TARGET}
-                            PRIVATE ${AZURE_SDK_LINK_LIBRARIES})
-    endforeach()
-  endif()
   if(ARROW_HDFS)
     foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
       target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE arrow::hadoop)

cpp/src/arrow/arrow-azure.pc.in

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+prefix=@CMAKE_INSTALL_PREFIX@
+includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@
+libdir=@ARROW_PKG_CONFIG_LIBDIR@
+
+Name: Apache Arrow Azure
+Description: Apache Arrow's Azure filesystem implementation.
+Version: @ARROW_VERSION@
+Requires: arrow@ARROW_AZURE_PC_REQUIRES@
+Requires.private:@ARROW_AZURE_PC_REQUIRES_PRIVATE@
+Libs: -L${libdir} -larrow_azure@ARROW_AZURE_PC_LIBS@
+Libs.private:@ARROW_AZURE_PC_LIBS_PRIVATE@
+Cflags:@ARROW_AZURE_PC_CFLAGS@
+Cflags.private:@ARROW_AZURE_PC_CFLAGS_PRIVATE@

cpp/src/arrow/filesystem/CMakeLists.txt

@@ -75,11 +75,29 @@ if(ARROW_GCS)
 endif()
 
 if(ARROW_AZURE)
+  set(ARROW_AZURE_TEST_EXTRA_LINK_LIBS)
+  if(ARROW_TEST_LINKAGE STREQUAL "shared")
+    list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS arrow_azure_shared)
+  else()
+    list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS arrow_azure_static)
+  endif()
+  list(APPEND ARROW_AZURE_TEST_EXTRA_LINK_LIBS ${AZURE_SDK_LINK_LIBRARIES})
   add_arrow_test(azurefs_test
                  EXTRA_LABELS
                  filesystem
                  EXTRA_LINK_LIBS
-                 ${AZURE_SDK_LINK_LIBRARIES})
+                 ${ARROW_AZURE_TEST_EXTRA_LINK_LIBS})
+  if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED)
+    add_arrow_test(azurefs_module_test
+                   SOURCES
+                   azurefs_module_test.cc
+                   EXTRA_LABELS
+                   filesystem
+                   DEFINITIONS
+                   ARROW_AZURE_LIBPATH="$<TARGET_FILE:arrow_azure_shared>"
+                   EXTRA_DEPENDENCIES
+                   arrow_azure_shared)
+  endif()
 endif()
 
 if(ARROW_S3)

cpp/src/arrow/filesystem/azure_visibility.h

@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#  if defined(_MSC_VER)
+#    pragma warning(push)
+#    pragma warning(disable : 4251)
+#  else
+#    pragma GCC diagnostic ignored "-Wattributes"
+#  endif
+
+#  ifdef ARROW_AZURE_STATIC
+#    define ARROW_AZURE_EXPORT
+#  elif defined(ARROW_AZURE_EXPORTING)
+#    define ARROW_AZURE_EXPORT __declspec(dllexport)
+#  else
+#    define ARROW_AZURE_EXPORT __declspec(dllimport)
+#  endif
+
+#  define ARROW_AZURE_NO_EXPORT
+
+#  if defined(_MSC_VER)
+#    pragma warning(pop)
+#  endif
+
+#else  // Not Windows
+#  ifndef ARROW_AZURE_EXPORT
+#    define ARROW_AZURE_EXPORT __attribute__((visibility("default")))
+#  endif
+#  ifndef ARROW_AZURE_NO_EXPORT
+#    define ARROW_AZURE_NO_EXPORT __attribute__((visibility("hidden")))
+#  endif
+#endif

cpp/src/arrow/filesystem/azurefs.cc

@@ -3561,4 +3561,15 @@ Result<std::string> AzureFileSystem::PathFromUri(const std::string& uri_string)
   return path;
 }
 
+static Result<std::shared_ptr<FileSystem>> AzureFileSystemFactory(
+    const Uri& uri, const io::IOContext& io_context, std::string* out_path) {
+  ARROW_ASSIGN_OR_RAISE(auto options, AzureOptions::FromUri(uri, out_path));
+  return AzureFileSystem::Make(options, io_context);
+}
+
+FileSystemRegistrar kAzureFileSystemRegistrar[]{
+    ARROW_REGISTER_FILESYSTEM("abfs", AzureFileSystemFactory, {}),
+    ARROW_REGISTER_FILESYSTEM("abfss", AzureFileSystemFactory, {}),
+};
+
 }  // namespace arrow::fs

cpp/src/arrow/filesystem/azurefs.h

@@ -21,6 +21,7 @@
 #include <string>
 #include <vector>
 
+#include "arrow/filesystem/azure_visibility.h"
 #include "arrow/filesystem/filesystem.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/uri.h"
@@ -59,7 +60,7 @@ class TestAzureOptions;
 /// - `AZURE_FEDERATED_TOKEN_FILE`
 ///
 /// Functions are provided for explicit configuration of credentials if that is preferred.
-struct ARROW_EXPORT AzureOptions {
+struct ARROW_AZURE_EXPORT AzureOptions {
   friend class TestAzureOptions;
 
   /// \brief The name of the Azure Storage Account being accessed.
@@ -247,7 +248,7 @@ struct ARROW_EXPORT AzureOptions {
 /// [2]: https://azure.microsoft.com/en-us/products/storage/data-lake-storage
 /// [3]:
 /// https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-known-issues
-class ARROW_EXPORT AzureFileSystem : public FileSystem {
+class ARROW_AZURE_EXPORT AzureFileSystem : public FileSystem {
  private:
   class Impl;
   std::unique_ptr<Impl> impl_;

cpp/src/arrow/filesystem/azurefs_internal.h

@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include "arrow/filesystem/azure_visibility.h"
 #include "arrow/result.h"
 
 namespace Azure::Storage::Files::DataLake {
@@ -70,7 +71,8 @@ enum class HierarchicalNamespaceSupport {
 /// account.
 /// \return kEnabled/kDisabled/kContainerNotFound (kUnknown is never
 /// returned).
-ARROW_EXPORT Result<HierarchicalNamespaceSupport> CheckIfHierarchicalNamespaceIsEnabled(
+ARROW_AZURE_EXPORT Result<HierarchicalNamespaceSupport>
+CheckIfHierarchicalNamespaceIsEnabled(
     const Azure::Storage::Files::DataLake::DataLakeFileSystemClient& adlfs_client,
     const arrow::fs::AzureOptions& options);
 

cpp/src/arrow/filesystem/azurefs_module.cc

@@ -0,0 +1,18 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "arrow/filesystem/filesystem_library.h"