From 1ceda8026ba05a731f541c9b94d725dca2a83e37 Mon Sep 17 00:00:00 2001 From: Ben Lasscock Date: Fri, 26 Jun 2026 19:40:02 +0000 Subject: [PATCH 1/2] Add opt-in monolithic shared library for ODR-safe plugin co-loading MDIO is consumed header-only, so every translation unit that touches it statically links its own copy of tensorstore and its vendored Abseil. That is harmless for a single executable, but a process that dlopen()s more than one such plugin ends up with several copies of Abseil's global state and aborts at runtime with the infamous "ODR violation in Cord". This adds an opt-in MDIO_BUILD_MONOLITHIC_SHARED (default OFF, so static consumers are untouched) that emits one libmdio_monolith.so, exposed via the mdio::monolith alias. Plugins link that single object instead of the tensorstore::* static deps, so the dynamic linker maps tensorstore and Abseil exactly once per process and the globals are singletons again. Getting a self-contained shared object right took two non-obvious steps: - WHOLE_ARCHIVE on the tensorstore::* targets is a no-op because they are INTERFACE aggregators. A small recursive collector walks the link closure down to the concrete STATIC_LIBRARY targets (unwrapping the $ genexes tensorstore uses for its alwayslink driver libs) and whole-archives those, so the explicit template instantiations (Spec / Zarr metadata JSON binders) and the driver self-registration objects are all force-included. - The interface deps are re-exposed to consumers as $ usage requirements, propagating the transitive tensorstore/Abseil/ nlohmann include dirs and defines without dragging the static archives back into the consumer (which would re-duplicate Abseil and defeat the whole exercise). Co-authored-by: Cursor --- mdio/CMakeLists.txt | 139 ++++++++++++++++++++++++++++++++++++++++++++ mdio/monolith.cc | 30 ++++++++++ 2 files changed, 169 insertions(+) create mode 100644 mdio/monolith.cc diff --git a/mdio/CMakeLists.txt b/mdio/CMakeLists.txt index 21e7599..ec896a0 100644 --- a/mdio/CMakeLists.txt +++ b/mdio/CMakeLists.txt @@ -110,6 +110,145 @@ install(EXPORT mdioTargets # ============ End installable library ============ +# ============ Optional monolithic shared library ============ +# +# By default mdio is consumed as a header-only INTERFACE target and each +# consumer statically links its own copy of tensorstore + Abseil. That is fine +# for a single executable, but a process that dynamically loads more than one +# such consumer (e.g. several plugins via dlopen) ends up with multiple copies +# of Abseil's global state and aborts at runtime ("ODR violation in Cord"). +# +# When MDIO_BUILD_MONOLITHIC_SHARED is enabled we additionally emit a single +# shared object that bundles tensorstore (and its vendored Abseil) once. +# Consumers that need ODR-safe co-loading link `mdio::monolith` instead of +# `mdio` + the tensorstore::* internal deps, so the dynamic linker maps +# tensorstore/Abseil exactly once per process. +option(MDIO_BUILD_MONOLITHIC_SHARED + "Also build libmdio_monolith.so bundling tensorstore+Abseil for ODR-safe co-loading of multiple mdio-linked plugins in one process" + OFF) + +# Recursively walk the link closure of the given targets and collect every +# concrete STATIC_LIBRARY target. The tensorstore::* entries are INTERFACE +# aggregators, so $ applied to them is a no-op; +# we need the real archives underneath so that explicit template instantiations +# (e.g. the tensorstore Spec / Zarr metadata JSON binders) and the driver +# self-registration objects are all force-included into the shared object. +# Dependency entries can be plain names, namespaced aliases (foo::bar), or +# genex-wrapped libraries such as +# $; we tokenize on +# genex punctuation (protecting :: in namespaced names) and keep whatever +# resolves to a real target. +function(_mdio_collect_static_archives outvar) + set(_result "") + set(_seen "") + set(_stack ${ARGN}) + while(_stack) + list(POP_FRONT _stack _t) + if(NOT TARGET ${_t}) + continue() + endif() + if(_t IN_LIST _seen) + continue() + endif() + list(APPEND _seen ${_t}) + get_target_property(_type ${_t} TYPE) + if(_type STREQUAL "STATIC_LIBRARY") + list(APPEND _result ${_t}) + endif() + set(_deps "") + get_target_property(_il ${_t} INTERFACE_LINK_LIBRARIES) + if(_il) + list(APPEND _deps ${_il}) + endif() + if(NOT _type STREQUAL "INTERFACE_LIBRARY") + get_target_property(_ll ${_t} LINK_LIBRARIES) + if(_ll) + list(APPEND _deps ${_ll}) + endif() + endif() + foreach(_d ${_deps}) + string(REPLACE "::" "@@NS@@" _d "${_d}") + string(REGEX REPLACE "[$<>:,]" ";" _toks "${_d}") + foreach(_tok ${_toks}) + string(REPLACE "@@NS@@" "::" _tok "${_tok}") + if(_tok AND TARGET ${_tok}) + list(APPEND _stack "${_tok}") + endif() + endforeach() + endforeach() + endwhile() + list(REMOVE_DUPLICATES _result) + set(${outvar} ${_result} PARENT_SCOPE) +endfunction() + +if(MDIO_BUILD_MONOLITHIC_SHARED) + # The tensorstore drivers self-register through static initializers, so they + # must be whole-archived into the shared object or the registrations get + # stripped (and zarr/s3/gcs stores fail to open at runtime). The remaining + # Abseil objects are pulled in transitively by normal symbol resolution, so + # they only appear once -- which is the whole point. + set(mdio_MONOLITH_DEPS + tensorstore::driver_array + tensorstore::driver_zarr + tensorstore::driver_zarr3 + tensorstore::driver_json + tensorstore::kvstore_file + tensorstore::kvstore_s3 + tensorstore::kvstore_gcs + tensorstore::stack + tensorstore::tensorstore + tensorstore::index_space_dim_expression + tensorstore::index_space_index_transform + tensorstore::util_status_testutil + nlohmann_json_schema_validator::nlohmann_json_schema_validator + ) + + add_library(mdio_monolith SHARED ${CMAKE_CURRENT_SOURCE_DIR}/monolith.cc) + set_target_properties(mdio_monolith PROPERTIES + OUTPUT_NAME mdio_monolith + POSITION_INDEPENDENT_CODE ON + ) + # Resolve the interface deps down to the concrete static archives and + # whole-archive every one of them so the shared object is self-contained. + _mdio_collect_static_archives(mdio_MONOLITH_ARCHIVES ${mdio_MONOLITH_DEPS}) + list(LENGTH mdio_MONOLITH_ARCHIVES _n_arch) + message(STATUS "MDIO monolith: whole-archiving ${_n_arch} static archive(s)") + + # The archives are whole-archived into this .so (PRIVATE link) so it is + # self-contained. The interface deps are ALSO re-exposed to consumers as + # COMPILE_ONLY usage requirements ($ requires CMake >= 3.27): + # this propagates the transitive tensorstore/Abseil/nlohmann include + # directories and compile definitions to anything linking mdio::monolith, + # WITHOUT pulling the static archives back into the consumer (which would + # re-duplicate Abseil globals and defeat the purpose). + target_link_libraries(mdio_monolith + PRIVATE + "$" + PUBLIC + "$" + ) + target_include_directories(mdio_monolith PUBLIC + $ + $ + ${TENSORSTORE_INCLUDE_DIRS} + ) + target_compile_definitions(mdio_monolith PUBLIC MAX_NUM_SLICES=${MAX_NUM_SLICES}) + target_compile_features(mdio_monolith PUBLIC cxx_std_17) + + # mdio::monolith is the public alias consumers link against. + add_library(mdio::monolith ALIAS mdio_monolith) + + install(TARGETS mdio_monolith EXPORT mdioTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + + message(STATUS "MDIO monolithic shared library enabled --> mdio::monolith") +endif() + +# ============ End optional monolithic shared library ============ + mdio_cc_test( NAME diff --git a/mdio/monolith.cc b/mdio/monolith.cc new file mode 100644 index 0000000..efd2251 --- /dev/null +++ b/mdio/monolith.cc @@ -0,0 +1,30 @@ +// Copyright 2026 TGS +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Translation-unit anchor for the optional monolithic shared library +// (MDIO_BUILD_MONOLITHIC_SHARED). The heavy tensorstore/Abseil objects -- and +// the driver self-registration initializers -- are force-included via +// WHOLE_ARCHIVE in CMake; this file just gives the shared object a concrete, +// exported symbol and pulls the public header so the include graph is built. +// +// Why this library exists: a process that loads more than one plugin which +// each statically embed tensorstore/Abseil ends up with multiple copies of +// Abseil's global state (e.g. the Cord registry), which aborts at runtime +// ("ODR violation in Cord"). Linking every such plugin against this single +// shared library instead means the dynamic linker maps tensorstore/Abseil +// exactly once, so those globals are singletons again. + +#include + +extern "C" const char* mdio_monolith_version() { return "1.0.0"; } From 4728cc78ef033c3142c20a5a7b452a57735ba904 Mon Sep 17 00:00:00 2001 From: Brian Michell Date: Thu, 9 Jul 2026 14:43:22 -0500 Subject: [PATCH 2/2] Set up a full monolithic installer for MDIO (#172) --- README.md | 5 + USER_GUIDE.md | 54 +++++++ .../Findnlohmann_json_schema_validator.cmake | 13 ++ cmake/MdioMonolithicShared.cmake | 126 ++++++++++++++++ cmake/MdioWholeArchiveLink.cmake | 91 ++++++++++++ cmake/mdioConfig.cmake.in | 10 +- mdio/CMakeLists.txt | 136 +++--------------- 7 files changed, 314 insertions(+), 121 deletions(-) create mode 100644 cmake/MdioMonolithicShared.cmake create mode 100644 cmake/MdioWholeArchiveLink.cmake diff --git a/README.md b/README.md index 715359c..f33fa69 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ Welcome to **MDIO** - a descriptive format for energy data that is intended to r **MDIO** schema definitions [here.](https://mdio-python.readthedocs.io/en/v1-new-schema/data_models/version_1.html) +# Recommended tools +- CMake 3.27 *or better* + # Requied tools - CMake 3.24 *or better* - A C++17 compiler @@ -64,6 +67,8 @@ $ make -j32 mdio_dataset_test ``` Each **MDIO** library will provide an associated cmake alias, e.g. mdio::mdio which can be use to link against **MDIO** in your project. +If you'd rather build **MDIO** once and reuse it across multiple projects via `find_package(mdio)` instead of `FetchContent`, see [Building and installing MDIO](USER_GUIDE.md#building-and-installing-mdio) in the User Guide. + ## API Documentation **MDIO** API documentation is currently provided with the **MDIO** library. diff --git a/USER_GUIDE.md b/USER_GUIDE.md index ee1e76b..bda2bb8 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -8,6 +8,10 @@ The goal of this user guide is to provide an introduction on how you may want to - [Linking](#linking) - [How to compile](#how-to-compile) - [What a full compile might look like for Hello, World!](#what-a-full-compile-might-look-like-for-hello-world) +- [Building and installing MDIO](#building-and-installing-mdio) + - [Standard install](#standard-install) + - [Monolithic shared library install](#monolithic-shared-library-install) + - [Consuming the installed package](#consuming-the-installed-package) - [Concepts](#concepts) - [Result based returns](#result-based-returns) - [Open options](#open-options) @@ -127,6 +131,56 @@ $ make -j$(nproc) hello_mdio $ ./hello_mdio ``` +## Building and installing MDIO +The sections above cover consuming **MDIO** with `FetchContent`, which re-fetches and rebuilds **MDIO** (and Tensorstore/Abseil) as part of every consuming project. If you want to build **MDIO** once and reuse it across multiple projects or container images, you can install it instead and consume it with `find_package(mdio)`. + +There are two installable forms: +- **Standard install** - the header-only `mdio::mdio` interface target, matching what `FetchContent` gives you in-tree. +- **Monolithic shared library install** - a single self-contained `libmdio_monolith.so` that bundles Tensorstore and Abseil, exposed as `mdio::monolith`. This is the recommended way to consume an installed **MDIO** from outside the `mdio-cpp` build tree, because it vendors the third-party headers it needs and requires no `FetchContent` network access at consume time. + +### Standard install +```BASH +$ mkdir build-install && cd build-install +$ cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/opt/mdio +$ cmake --build . -j$(nproc) --target install +``` +This installs the **MDIO** headers, the `mdio::mdio` interface target, and a CMake package config under `/opt/mdio/lib/cmake/mdio`. Because `mdio::mdio` is header-only, a consumer still needs to independently provide the same Tensorstore driver targets **MDIO** was built against (as described in [Linking](#linking)), so this form is best suited to projects that already vendor Tensorstore themselves. + +### Monolithic shared library install +The monolithic build requires CMake 3.27 *or better*. +```BASH +$ cmake -S . -B build-monolith \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PWD/install" \ + -DMDIO_BUILD_MONOLITHIC_SHARED=ON +$ cmake --build build-monolith --target install -j"$(nproc)" +``` +This installs: +- `lib/libmdio_monolith.so` - a single shared object with Tensorstore, Abseil, and the Zarr/file/S3/GCS drivers whole-archived in. Consumers link only this one library, so a process that loads more than one **MDIO**-linked plugin (e.g. via `dlopen`) still gets exactly one copy of Abseil's global state instead of aborting with an ODR violation. +- `include/mdio` - the **MDIO** headers. +- `include/{tensorstore,absl,riegeli,nlohmann_json,half}-src/...` - the third-party headers **MDIO**'s public headers depend on, vendored so consumers don't need their own copies or network access. +- `lib/cmake/mdio` - the CMake package config, including the `mdio::monolith` target. + +The maximum number of slices (`MAX_NUM_SLICES`, see [Slicing](#slicing)) is baked into the installed package at install time. Pass `-DMAX_NUM_SLICES=64` (or whatever value you need) to the first `cmake` invocation above if the default of 32 isn't enough; consumers pick it up automatically and do not need to redefine it. + +### Consuming the installed package +Point `CMAKE_PREFIX_PATH` at your install directory (skip this if you installed to a standard system prefix), then `find_package(mdio)` and link `mdio::monolith`: +```Cmake +cmake_minimum_required(VERSION 3.27) +project(hello_mdio_installed VERSION 1.0.0 LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 17) + +find_package(mdio REQUIRED) + +add_executable(hello_mdio src/hello_mdio.cc) +target_link_libraries(hello_mdio PRIVATE mdio::monolith) +``` +```BASH +$ cmake -S . -B build -DCMAKE_PREFIX_PATH=/path/to/mdio/install +$ cmake --build build -j$(nproc) +``` +No manual `-I` include paths, `MAX_NUM_SLICES` defines, or Tensorstore driver targets are needed; `mdio::monolith` carries all of it as usage requirements. + ## Concepts ### Result based returns **MDIO** aims to follow the Google style of [not throwing exceptions](https://google.github.io/styleguide/cppguide.html#Exceptions). Instead, we use result based returns wherever an error state could exist. A trivial example of this design pattern is a simple function that tries to divide two integers, and handles the case of divide-by-zero. diff --git a/cmake/Findnlohmann_json_schema_validator.cmake b/cmake/Findnlohmann_json_schema_validator.cmake index 141d61e..d8c2b48 100644 --- a/cmake/Findnlohmann_json_schema_validator.cmake +++ b/cmake/Findnlohmann_json_schema_validator.cmake @@ -27,10 +27,23 @@ if (NOT TARGET nlohmann_json_schema_validator) target_link_libraries(nlohmann_json_schema_validator INTERFACE nlohmann_json::nlohmann_json) + # The upstream target carries a PUBLIC_HEADER property set to the relative + # path "nlohmann/json-schema.hpp". When we call install(TARGETS ...) from the + # mdio subdirectory, CMake resolves that relative path against the mdio source + # dir (.../mdio/nlohmann/json-schema.hpp, which does not exist) and would also + # flatten it to include/json-schema.hpp, breaking #include . + # Clear it and install the header explicitly, preserving the nlohmann/ prefix. + set_target_properties(nlohmann_json_schema_validator PROPERTIES PUBLIC_HEADER "") + # Install the validator target install(TARGETS nlohmann_json_schema_validator EXPORT mdioTargets ) + install(FILES + "${nlohmann_json_schema_validator_SOURCE_DIR}/src/nlohmann/json-schema.hpp" + DESTINATION include/nlohmann + ) + message(STATUS "Found json schema validator library") endif() diff --git a/cmake/MdioMonolithicShared.cmake b/cmake/MdioMonolithicShared.cmake new file mode 100644 index 0000000..bb9d2af --- /dev/null +++ b/cmake/MdioMonolithicShared.cmake @@ -0,0 +1,126 @@ +# Copyright 2026 TGS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Optional monolithic shared library (MDIO_BUILD_MONOLITHIC_SHARED). +# Requires CMake 3.27+ for $ link propagation. The archive +# de-duplication needed for the whole-archive link is handled by our own link +# wrapper (MdioWholeArchiveLink.cmake), which works with the default GNU ld / +# gold toolchain -- CMake's built-in CMP0156/CMP0179 de-duplication only applies +# to symbol-recording linkers such as LLD, so we cannot depend on it here. +cmake_minimum_required(VERSION 3.27) + +# The tensorstore drivers self-register through static initializers, so they +# must be whole-archived into the shared object or the registrations get +# stripped (and zarr/s3/gcs stores fail to open at runtime). The remaining +# Abseil objects are pulled in transitively by normal symbol resolution, so +# they only appear once -- which is the whole point. +set(mdio_MONOLITH_DEPS + tensorstore::driver_array + tensorstore::driver_zarr + tensorstore::driver_zarr3 + tensorstore::driver_json + tensorstore::kvstore_file + tensorstore::kvstore_s3 + tensorstore::kvstore_gcs + tensorstore::stack + tensorstore::tensorstore + tensorstore::index_space_dim_expression + tensorstore::index_space_index_transform + tensorstore::util_status_testutil + nlohmann_json_schema_validator::nlohmann_json_schema_validator +) + +# Route the .so link through MdioWholeArchiveLink.cmake so the entire static +# dependency closure is whole-archived into a single self-contained .so. See +# that file for the full rationale. +set(CMAKE_CXX_CREATE_SHARED_LIBRARY + "\"${CMAKE_COMMAND}\" -P \"${CMAKE_CURRENT_LIST_DIR}/MdioWholeArchiveLink.cmake\" -- -o ") + +add_library(mdio_monolith SHARED ${CMAKE_CURRENT_SOURCE_DIR}/monolith.cc) +set_target_properties(mdio_monolith PROPERTIES + OUTPUT_NAME mdio_monolith + POSITION_INDEPENDENT_CODE ON +) +# Link the full tensorstore closure into the .so. tensorstore INTERFACE targets +# already carry $ for their .alwayslink driver +# objects; the blanket whole-archive above subsumes that (harmless overlap). +target_link_libraries(mdio_monolith PRIVATE ${mdio_MONOLITH_DEPS}) +target_include_directories(mdio_monolith PUBLIC + $ + $ + ${TENSORSTORE_INCLUDE_DIRS} +) +target_compile_definitions(mdio_monolith PUBLIC MAX_NUM_SLICES=${MAX_NUM_SLICES}) +target_compile_features(mdio_monolith PUBLIC cxx_std_17) + +# Re-expose tensorstore/Abseil/nlohmann compile usage to consumers without +# pulling static archives back into them (ODR-safe co-loading). Kept on a +# separate INTERFACE target so COMPILE_ONLY deps never share a target with the +# .so's PRIVATE link closure. +# +# The COMPILE_ONLY deps reference tensorstore::* targets, which only exist in +# this build tree -- an installed consumer resolves headers from the vendored +# trees below instead. Wrap them in BUILD_INTERFACE so the exported target does +# not reference targets the consumer's project has never defined. +add_library(mdio_monolith_interface INTERFACE) +target_link_libraries(mdio_monolith_interface INTERFACE + mdio_monolith + "$>" +) + +# mdio::monolith is the public alias consumers link against in-tree; the +# installed package re-creates it in mdioConfig.cmake (aliases are not exported). +add_library(mdio::monolith ALIAS mdio_monolith_interface) + +# ---- Vendor the third-party headers needed to compile against mdio.h -------- +# mdio's public headers include tensorstore/absl/riegeli/half/nlohmann headers +# directly, so a consumer of the installed monolith needs those header trees. +# They are laid out under include/-src[/...] (matching the historical +# mdio-cpp-installer layout) and wired onto the target's INSTALL_INTERFACE so +# find_package(mdio) consumers get them automatically (no manual -I needed). +set(_mdio_deps_dir "${FETCHCONTENT_BASE_DIR}") +if(NOT _mdio_deps_dir) + set(_mdio_deps_dir "${CMAKE_BINARY_DIR}/_deps") +endif() + +install(DIRECTORY "${_mdio_deps_dir}/tensorstore-src/tensorstore" + DESTINATION include/tensorstore-src + FILES_MATCHING PATTERN "*.h" PATTERN "*.inc") +install(DIRECTORY "${_mdio_deps_dir}/absl-src/absl" + DESTINATION include/absl-src + FILES_MATCHING PATTERN "*.h" PATTERN "*.inc") +install(DIRECTORY "${_mdio_deps_dir}/riegeli-src/riegeli" + DESTINATION include/riegeli-src + FILES_MATCHING PATTERN "*.h" PATTERN "*.inc") +install(DIRECTORY "${_mdio_deps_dir}/nlohmann_json-src/include/nlohmann" + DESTINATION include/nlohmann_json-src/include + FILES_MATCHING PATTERN "*.hpp") +install(FILES "${_mdio_deps_dir}/half-src/include/half.hpp" + DESTINATION include/half-src/include) + +target_include_directories(mdio_monolith INTERFACE + $ + $ + $ + $ + $ +) + +install(TARGETS mdio_monolith mdio_monolith_interface EXPORT mdioTargets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} +) + +message(STATUS "MDIO monolithic shared library enabled --> mdio::monolith") diff --git a/cmake/MdioWholeArchiveLink.cmake b/cmake/MdioWholeArchiveLink.cmake new file mode 100644 index 0000000..70a8acf --- /dev/null +++ b/cmake/MdioWholeArchiveLink.cmake @@ -0,0 +1,91 @@ +# Copyright 2026 TGS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Link wrapper used by MdioMonolithicShared.cmake to build libmdio_monolith.so. +# +# It is invoked as: +# cmake -P MdioWholeArchiveLink.cmake -- -o \ +# +# +# and rewrites the link command so the .so contains the *entire* static +# dependency closure: +# * every static archive (*.a) is de-duplicated, keeping the first occurrence. +# CMake lists some archives twice for circular-dependency resolution with +# traditional linkers (GNU ld / gold); under --whole-archive that would cause +# "multiple definition" errors. (CMake's own CMP0156/CMP0179 de-duplication +# only kicks in for symbol-recording linkers such as LLD, so we cannot rely +# on it with the default toolchain.) +# * all archives are bracketed with --whole-archive/--no-whole-archive so +# objects the .so's own code never references are still included. A consumer +# that instantiates a tensorstore/Abseil/riegeli template from a header then +# resolves the out-of-line symbol from the .so instead of failing to link. +# * shared/system libraries (-l...) are placed after --no-whole-archive so the +# compiler-implicit trailing libraries (e.g. libgcc) are never whole-archived. +# +# This wrapper is generic (it does not encode anything tensorstore-specific), so +# it needs no maintenance as the dependency graph evolves. + +# Collect the arguments that follow the "--" sentinel. +set(_args "") +set(_seen_sep FALSE) +math(EXPR _last "${CMAKE_ARGC} - 1") +foreach(_i RANGE 0 ${_last}) + set(_a "${CMAKE_ARGV${_i}}") + if(_seen_sep) + list(APPEND _args "${_a}") + elseif(_a STREQUAL "--") + set(_seen_sep TRUE) + endif() +endforeach() + +if(NOT _seen_sep) + message(FATAL_ERROR "MdioWholeArchiveLink: missing '--' argument separator") +endif() + +set(_head "") # compiler, flags, -o , objects, misc linker flags +set(_archives "") # unique static archives, in first-seen order +set(_libs "") # -l... shared/system libraries (kept after the archives) +set(_seen_archives "") + +foreach(_a IN LISTS _args) + if(_a STREQUAL "-Wl,--push-state,--whole-archive" OR _a STREQUAL "-Wl,--pop-state") + # Drop tensorstore's per-archive whole-archive wrappers; the global bracket + # below subsumes them and the stray push/pop would otherwise be unbalanced + # once archives are reordered. + continue() + elseif(_a MATCHES "\\.a$") + list(FIND _seen_archives "${_a}" _found) + if(_found EQUAL -1) + list(APPEND _seen_archives "${_a}") + list(APPEND _archives "${_a}") + endif() + elseif(_a MATCHES "^-l") + list(APPEND _libs "${_a}") + else() + list(APPEND _head "${_a}") + endif() +endforeach() + +set(_cmd + ${_head} + -Wl,--whole-archive + ${_archives} + -Wl,--no-whole-archive + ${_libs} +) + +execute_process(COMMAND ${_cmd} RESULT_VARIABLE _rc) +if(NOT _rc EQUAL 0) + message(FATAL_ERROR "MdioWholeArchiveLink: link command failed (${_rc})") +endif() diff --git a/cmake/mdioConfig.cmake.in b/cmake/mdioConfig.cmake.in index fa90574..09eff31 100644 --- a/cmake/mdioConfig.cmake.in +++ b/cmake/mdioConfig.cmake.in @@ -1,3 +1,11 @@ @PACKAGE_INIT@ -include("${CMAKE_CURRENT_LIST_DIR}/mdioTargets.cmake") \ No newline at end of file +include("${CMAKE_CURRENT_LIST_DIR}/mdioTargets.cmake") + +# Aliases are not part of an export set, so re-create the public monolith alias +# for installed consumers. Link mdio::monolith for ODR-safe co-loading. +if(TARGET mdio::mdio_monolith_interface AND NOT TARGET mdio::monolith) + add_library(mdio::monolith INTERFACE IMPORTED) + set_target_properties(mdio::monolith PROPERTIES + INTERFACE_LINK_LIBRARIES mdio::mdio_monolith_interface) +endif() \ No newline at end of file diff --git a/mdio/CMakeLists.txt b/mdio/CMakeLists.txt index ec896a0..08ea17f 100644 --- a/mdio/CMakeLists.txt +++ b/mdio/CMakeLists.txt @@ -57,8 +57,12 @@ target_link_libraries(mdio INTERFACE # Install the mdio target install(TARGETS mdio EXPORT mdioTargets) -# Install include directories -install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION include/mdio) +# Install include directories (headers only -- skip test .cc, .py, CMake files) +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ DESTINATION include/mdio + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.hpp" +) # Ensure the original nlohmann_json target is part of the export set if (TARGET nlohmann_json) @@ -123,128 +127,20 @@ install(EXPORT mdioTargets # Consumers that need ODR-safe co-loading link `mdio::monolith` instead of # `mdio` + the tensorstore::* internal deps, so the dynamic linker maps # tensorstore/Abseil exactly once per process. +# +# Implementation lives in cmake/MdioMonolithicShared.cmake and requires +# CMake 3.27+ ($ link propagation). option(MDIO_BUILD_MONOLITHIC_SHARED - "Also build libmdio_monolith.so bundling tensorstore+Abseil for ODR-safe co-loading of multiple mdio-linked plugins in one process" + "Also build libmdio_monolith.so bundling tensorstore+Abseil for ODR-safe co-loading of multiple mdio-linked plugins in one process (requires CMake 3.27+)" OFF) -# Recursively walk the link closure of the given targets and collect every -# concrete STATIC_LIBRARY target. The tensorstore::* entries are INTERFACE -# aggregators, so $ applied to them is a no-op; -# we need the real archives underneath so that explicit template instantiations -# (e.g. the tensorstore Spec / Zarr metadata JSON binders) and the driver -# self-registration objects are all force-included into the shared object. -# Dependency entries can be plain names, namespaced aliases (foo::bar), or -# genex-wrapped libraries such as -# $; we tokenize on -# genex punctuation (protecting :: in namespaced names) and keep whatever -# resolves to a real target. -function(_mdio_collect_static_archives outvar) - set(_result "") - set(_seen "") - set(_stack ${ARGN}) - while(_stack) - list(POP_FRONT _stack _t) - if(NOT TARGET ${_t}) - continue() - endif() - if(_t IN_LIST _seen) - continue() - endif() - list(APPEND _seen ${_t}) - get_target_property(_type ${_t} TYPE) - if(_type STREQUAL "STATIC_LIBRARY") - list(APPEND _result ${_t}) - endif() - set(_deps "") - get_target_property(_il ${_t} INTERFACE_LINK_LIBRARIES) - if(_il) - list(APPEND _deps ${_il}) - endif() - if(NOT _type STREQUAL "INTERFACE_LIBRARY") - get_target_property(_ll ${_t} LINK_LIBRARIES) - if(_ll) - list(APPEND _deps ${_ll}) - endif() - endif() - foreach(_d ${_deps}) - string(REPLACE "::" "@@NS@@" _d "${_d}") - string(REGEX REPLACE "[$<>:,]" ";" _toks "${_d}") - foreach(_tok ${_toks}) - string(REPLACE "@@NS@@" "::" _tok "${_tok}") - if(_tok AND TARGET ${_tok}) - list(APPEND _stack "${_tok}") - endif() - endforeach() - endforeach() - endwhile() - list(REMOVE_DUPLICATES _result) - set(${outvar} ${_result} PARENT_SCOPE) -endfunction() - if(MDIO_BUILD_MONOLITHIC_SHARED) - # The tensorstore drivers self-register through static initializers, so they - # must be whole-archived into the shared object or the registrations get - # stripped (and zarr/s3/gcs stores fail to open at runtime). The remaining - # Abseil objects are pulled in transitively by normal symbol resolution, so - # they only appear once -- which is the whole point. - set(mdio_MONOLITH_DEPS - tensorstore::driver_array - tensorstore::driver_zarr - tensorstore::driver_zarr3 - tensorstore::driver_json - tensorstore::kvstore_file - tensorstore::kvstore_s3 - tensorstore::kvstore_gcs - tensorstore::stack - tensorstore::tensorstore - tensorstore::index_space_dim_expression - tensorstore::index_space_index_transform - tensorstore::util_status_testutil - nlohmann_json_schema_validator::nlohmann_json_schema_validator - ) - - add_library(mdio_monolith SHARED ${CMAKE_CURRENT_SOURCE_DIR}/monolith.cc) - set_target_properties(mdio_monolith PROPERTIES - OUTPUT_NAME mdio_monolith - POSITION_INDEPENDENT_CODE ON - ) - # Resolve the interface deps down to the concrete static archives and - # whole-archive every one of them so the shared object is self-contained. - _mdio_collect_static_archives(mdio_MONOLITH_ARCHIVES ${mdio_MONOLITH_DEPS}) - list(LENGTH mdio_MONOLITH_ARCHIVES _n_arch) - message(STATUS "MDIO monolith: whole-archiving ${_n_arch} static archive(s)") - - # The archives are whole-archived into this .so (PRIVATE link) so it is - # self-contained. The interface deps are ALSO re-exposed to consumers as - # COMPILE_ONLY usage requirements ($ requires CMake >= 3.27): - # this propagates the transitive tensorstore/Abseil/nlohmann include - # directories and compile definitions to anything linking mdio::monolith, - # WITHOUT pulling the static archives back into the consumer (which would - # re-duplicate Abseil globals and defeat the purpose). - target_link_libraries(mdio_monolith - PRIVATE - "$" - PUBLIC - "$" - ) - target_include_directories(mdio_monolith PUBLIC - $ - $ - ${TENSORSTORE_INCLUDE_DIRS} - ) - target_compile_definitions(mdio_monolith PUBLIC MAX_NUM_SLICES=${MAX_NUM_SLICES}) - target_compile_features(mdio_monolith PUBLIC cxx_std_17) - - # mdio::monolith is the public alias consumers link against. - add_library(mdio::monolith ALIAS mdio_monolith) - - install(TARGETS mdio_monolith EXPORT mdioTargets - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) - - message(STATUS "MDIO monolithic shared library enabled --> mdio::monolith") + if(CMAKE_VERSION VERSION_LESS "3.27") + message(FATAL_ERROR + "MDIO_BUILD_MONOLITHIC_SHARED requires CMake 3.27 or later " + "(uses $ link feature).") + endif() + include(${CMAKE_CURRENT_SOURCE_DIR}/../cmake/MdioMonolithicShared.cmake) endif() # ============ End optional monolithic shared library ============