From e31a97d39f0700d3243cb90e1d0a99966eb5fbe5 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 15:54:16 +0800 Subject: [PATCH 01/27] feat(cpp): add native installation packages --- .github/workflows/cpp-packaging.yml | 130 +++++++++++++ cpp/CMakeLists.txt | 171 +++++++++++++++++- cpp/cmake/TsFileConfig.cmake.in | 29 +++ cpp/cmake/TsFilePublicHeaders.cmake | 47 +++++ cpp/cmake/libtsfile.pc.in | 31 ++++ .../projects/ANTLR4Dependency/CMakeLists.txt | 2 +- .../projects/InstalledConsumer/CMakeLists.txt | 42 +++++ .../tests/projects/InstalledConsumer/main.cc | 24 +++ .../projects/PkgConfigConsumer/CMakeLists.txt | 32 ++++ .../tests/projects/PkgConfigConsumer/main.c | 21 +++ .../tests/projects/PkgConfigConsumer/main.cc | 21 +++ cpp/src/CMakeLists.txt | 118 ++++++++++-- cpp/third_party/CMakeLists.txt | 23 +++ cpp/tools/CMakeLists.txt | 17 +- packaging/README.md | 92 ++++++++++ packaging/homebrew/tsfile.rb | 71 ++++++++ 16 files changed, 853 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/cpp-packaging.yml create mode 100644 cpp/cmake/TsFileConfig.cmake.in create mode 100644 cpp/cmake/TsFilePublicHeaders.cmake create mode 100644 cpp/cmake/libtsfile.pc.in create mode 100644 cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt create mode 100644 cpp/cmake/tests/projects/InstalledConsumer/main.cc create mode 100644 cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt create mode 100644 cpp/cmake/tests/projects/PkgConfigConsumer/main.c create mode 100644 cpp/cmake/tests/projects/PkgConfigConsumer/main.cc create mode 100644 packaging/README.md create mode 100644 packaging/homebrew/tsfile.rb diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml new file mode 100644 index 000000000..ef5a457c2 --- /dev/null +++ b/.github/workflows/cpp-packaging.yml @@ -0,0 +1,130 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Cpp-Packaging + +on: + push: + branches: + - develop + - rc/** + paths: + - '.github/workflows/cpp-packaging.yml' + - 'cpp/**' + - 'packaging/**' + - 'LICENSE' + - 'NOTICE' + pull_request: + branches: + - develop + - rc/** + paths: + - '.github/workflows/cpp-packaging.yml' + - 'cpp/**' + - 'packaging/**' + - 'LICENSE' + - 'NOTICE' + workflow_dispatch: + +permissions: + contents: read + +jobs: + deb: + name: Debian package + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install packaging tools + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ninja-build pkg-config dpkg-dev + + - name: Configure and build + run: | + cmake -S cpp -B build/deb -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO \ + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + cmake --build build/deb --parallel + cmake --install build/deb + + - name: Build and inspect DEB packages + run: | + cpack --config build/deb/CPackConfig.cmake -G DEB + ls -lh ./*.deb + for package in ./*.deb; do + dpkg-deb --info "$package" + dpkg-deb --contents "$package" | grep -E '(/libtsfile|/tsfile-cli|TsFileConfig|libtsfile.pc)' || true + done + + - name: Upload DEB packages + uses: actions/upload-artifact@v4 + with: + name: tsfile-deb + path: '*.deb' + if-no-files-found: error + + rpm: + name: Fedora package + runs-on: ubuntu-24.04 + container: fedora:latest + timeout-minutes: 30 + steps: + - name: Install build and packaging tools + run: | + dnf install -y \ + cmake gcc-c++ make ninja-build pkgconf-pkg-config rpm-build \ + git curl tar xz unzip gzip + + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Configure and build + run: | + cmake -S cpp -B build/rpm -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO \ + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + cmake --build build/rpm --parallel + cmake --install build/rpm + + - name: Build and inspect RPM packages + run: | + cpack --config build/rpm/CPackConfig.cmake -G RPM + ls -lh ./*.rpm + for package in ./*.rpm; do + rpm -qip "$package" + rpm -qlp "$package" | grep -E '(/libtsfile|/tsfile-cli|TsFileConfig|libtsfile.pc)' || true + done + + - name: Upload RPM packages + uses: actions/upload-artifact@v4 + with: + name: tsfile-rpm + path: '*.rpm' + if-no-files-found: error diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c39bcbc28..567286fa5 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -34,6 +34,25 @@ if (POLICY CMP0074) endif () set(TsFile_CPP_VERSION 2.3.2.dev) +include(GNUInstallDirs) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/TsFilePublicHeaders.cmake) + +# The package version identifies the source release. Development suffixes are +# intentionally removed from generated package metadata and never enter the +# SONAME. +string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" TSFILE_PACKAGE_VERSION + "${TsFile_CPP_VERSION}") +if ("${TSFILE_PACKAGE_VERSION}" STREQUAL "") + message(FATAL_ERROR + "TsFile_CPP_VERSION must start with a semantic version: " + "${TsFile_CPP_VERSION}") +endif () +set(TSFILE_ABI_VERSION "1" CACHE STRING + "TsFile shared-library ABI epoch (changes only on ABI breaks)") +if (NOT TSFILE_ABI_VERSION MATCHES "^[0-9]+$") + message(FATAL_ERROR "TSFILE_ABI_VERSION must be a positive integer") +endif () + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/DependencySource.cmake) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -64,10 +83,18 @@ endif () message("cmake using: USE_CPP11=${USE_CPP11}") # MSVC has no /std:c++11; CMake maps this to the closest supported standard # (C++14 default on MSVC), which compiles the C++11 codebase fine. -set(CMAKE_CXX_STANDARD 11) +set(TSFILE_CXX_STANDARD "11" CACHE STRING + "C++ language standard used to build TsFile (11, 14, or 17)") +set_property(CACHE TSFILE_CXX_STANDARD PROPERTY STRINGS 11 14 17) +if (NOT TSFILE_CXX_STANDARD MATCHES "^(11|14|17)$") + message(FATAL_ERROR + "TSFILE_CXX_STANDARD must be one of 11, 14, or 17") +endif () +set(CMAKE_CXX_STANDARD ${TSFILE_CXX_STANDARD}) set(CMAKE_CXX_STANDARD_REQUIRED OFF) if (NOT MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -std=c++${TSFILE_CXX_STANDARD}") endif () if (DEFINED ENV{CXX}) @@ -336,7 +363,8 @@ if (MSVC) # C++17 extensions), so we pin it explicitly for reproducibility. set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} /W3 /utf-8 /EHsc /bigobj /Zc:__cplusplus /std:c++14") else () - set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -std=c++11") + set(CMAKE_CXX_FLAGS + "$ENV{CXXFLAGS} -Wall -std=c++${TSFILE_CXX_STANDARD}") endif () add_subdirectory(third_party) @@ -418,3 +446,140 @@ endif () unset(_TSFILE_PROJECT_DEPENDENCIES) add_subdirectory(examples) + +# Install the compatibility header closure from the generated staging tree. The +# staging tree is populated by the existing copy_* targets and contains only +# headers selected by the reviewed closure manifest. The closure is retained +# for source compatibility; it is not the final public API boundary. +foreach (header_dir IN LISTS TSFILE_PUBLIC_HEADER_CLOSURE) + install(DIRECTORY "${LIBRARY_INCLUDE_DIR}/${header_dir}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tsfile/${header_dir}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endforeach() +if (ENABLE_SIMD AND TSFILE_SIMDE_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_SIMDE_INCLUDE_ROOT}/simde/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/simde" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_ANTLR4_INCLUDE_ROOT}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () +if (ENABLE_ANTLR4 AND TSFILE_UTF8CPP_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_UTF8CPP_INCLUDE_ROOT}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () + +set(TSFILE_LICENSE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE") +set(TSFILE_NOTICE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../NOTICE") +if (EXISTS "${TSFILE_LICENSE_FILE}") + install(FILES "${TSFILE_LICENSE_FILE}" + DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/tsfile" + COMPONENT runtime) +endif() +if (EXISTS "${TSFILE_NOTICE_FILE}") + install(FILES "${TSFILE_NOTICE_FILE}" + DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/tsfile" + COMPONENT runtime) +endif() + +include(CMakePackageConfigHelpers) +set(TSFILE_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile") +configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TsFileConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + INSTALL_DESTINATION "${TSFILE_CMAKE_INSTALL_DIR}") +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfigVersion.cmake" + VERSION "${TSFILE_PACKAGE_VERSION}" + COMPATIBILITY SameMajorVersion) +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfigVersion.cmake" + DESTINATION "${TSFILE_CMAKE_INSTALL_DIR}" + COMPONENT development) + +set(TSFILE_PKGCONFIG_CFLAGS "") +foreach (_TSFILE_PUBLIC_FEATURE + ENABLE_ANTLR4 + ENABLE_MEM_STAT + ENABLE_SNAPPY + ENABLE_LZ4 + ENABLE_LZOKAY + ENABLE_ZLIB + ENABLE_GZIP + ENABLE_ZSTD + ENABLE_LZMA2 + ENABLE_THREADS + ENABLE_SIMD) + if (${_TSFILE_PUBLIC_FEATURE}) + string(APPEND TSFILE_PKGCONFIG_CFLAGS + " -D${_TSFILE_PUBLIC_FEATURE}") + endif () +endforeach () +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libtsfile.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/libtsfile.pc" + @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libtsfile.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" + COMPONENT development) +unset(_TSFILE_PUBLIC_FEATURE) + +# CPack is an opt-in local/package-builder integration. Native release jobs +# can configure the project with TSFILE_DEPENDENCY_SOURCE=SYSTEM when the +# target image provides every compatible dependency; AUTO remains the +# reproducible fallback for distribution images with older packages. +option(TSFILE_ENABLE_CPACK + "Enable CPack DEB/RPM package metadata generation" OFF) +if (TSFILE_ENABLE_CPACK) + set(CPACK_PACKAGE_NAME "tsfile") + set(CPACK_PACKAGE_VENDOR "Apache Software Foundation") + set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Apache TsFile C++ library") + set(CPACK_PACKAGE_DESCRIPTION + "Columnar storage library and command-line tools for time series data") + set(CPACK_PACKAGE_HOMEPAGE_URL "https://tsfile.apache.org/") + set(CPACK_PACKAGE_CONTACT "dev@tsfile.apache.org") + set(CPACK_RESOURCE_FILE_LICENSE "${TSFILE_LICENSE_FILE}") + set(CPACK_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}") + string(REPLACE "." ";" _TSFILE_PACKAGE_VERSION_PARTS + "${TSFILE_PACKAGE_VERSION}") + list(GET _TSFILE_PACKAGE_VERSION_PARTS 0 CPACK_PACKAGE_VERSION_MAJOR) + list(GET _TSFILE_PACKAGE_VERSION_PARTS 1 CPACK_PACKAGE_VERSION_MINOR) + list(GET _TSFILE_PACKAGE_VERSION_PARTS 2 CPACK_PACKAGE_VERSION_PATCH) + + set(CPACK_COMPONENTS_ALL runtime development tools) + set(CPACK_COMPONENT_DEVELOPMENT_DEPENDS runtime) + set(CPACK_COMPONENT_TOOLS_DEPENDS runtime) + set(CPACK_DEB_COMPONENT_INSTALL ON) + set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) + set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) + set(CPACK_DEBIAN_RUNTIME_PACKAGE_NAME "tsfile") + set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_NAME "tsfile-dev") + set(CPACK_DEBIAN_TOOLS_PACKAGE_NAME "tsfile-tools") + set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_DEPENDS + "tsfile (= ${TSFILE_PACKAGE_VERSION})") + set(CPACK_DEBIAN_TOOLS_PACKAGE_DEPENDS + "tsfile (= ${TSFILE_PACKAGE_VERSION})") + + set(CPACK_RPM_COMPONENT_INSTALL ON) + set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") + set(CPACK_RPM_PACKAGE_RELEASE "1") + set(CPACK_RPM_PACKAGE_AUTOREQPROV ON) + set(CPACK_RPM_RUNTIME_PACKAGE_NAME "tsfile") + set(CPACK_RPM_DEVELOPMENT_PACKAGE_NAME "tsfile-devel") + set(CPACK_RPM_TOOLS_PACKAGE_NAME "tsfile-tools") + set(CPACK_RPM_DEVELOPMENT_PACKAGE_REQUIRES + "tsfile = ${TSFILE_PACKAGE_VERSION}-1") + set(CPACK_RPM_TOOLS_PACKAGE_REQUIRES + "tsfile = ${TSFILE_PACKAGE_VERSION}-1") + + include(CPack) + unset(_TSFILE_PACKAGE_VERSION_PARTS) +endif () diff --git a/cpp/cmake/TsFileConfig.cmake.in b/cpp/cmake/TsFileConfig.cmake.in new file mode 100644 index 000000000..eea711a9e --- /dev/null +++ b/cpp/cmake/TsFileConfig.cmake.in @@ -0,0 +1,29 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +if (@ENABLE_THREADS@) + find_dependency(Threads) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/TsFileTargets.cmake") + +check_required_components(TsFile) diff --git a/cpp/cmake/TsFilePublicHeaders.cmake b/cpp/cmake/TsFilePublicHeaders.cmake new file mode 100644 index 000000000..a517d3a9c --- /dev/null +++ b/cpp/cmake/TsFilePublicHeaders.cmake @@ -0,0 +1,47 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# Public entry points are intentionally listed separately from the transitive +# header closure. The current public headers include implementation-level +# declarations, so the closure is installed until those dependencies can be +# hidden behind a stable facade. Adding a new public entry point requires an +# explicit review of this list. +set(TSFILE_PUBLIC_ENTRYPOINT_HEADERS + cwrapper/tsfile_cwrapper.h + cwrapper/tsfile_cwrapper_expression.h + reader/tsfile_reader.h + writer/tsfile_writer.h + writer/tsfile_table_writer.h + writer/tsfile_tree_writer.h) + +# These directories form the reviewed transitive closure of the entry points +# above. This is an installation compatibility closure, not a promise that +# every header is a stable public API. Compression, encoding, parser, and +# utility headers are implementation dependencies today, but are required for +# consumers to parse the installed public declarations. +set(TSFILE_PUBLIC_HEADER_CLOSURE + common + compress + cwrapper + encoding + file + parser + reader + utils + writer) diff --git a/cpp/cmake/libtsfile.pc.in b/cpp/cmake/libtsfile.pc.in new file mode 100644 index 000000000..ff820d0d8 --- /dev/null +++ b/cpp/cmake/libtsfile.pc.in @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The .pc file is installed below ${prefix}/@CMAKE_INSTALL_LIBDIR@/pkgconfig. +# Deriving the prefix from its own location keeps DESTDIR and relocations +# working without embedding the build-time install prefix. +prefix=${pcfiledir}/../.. +exec_prefix=${prefix} +libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: libtsfile +Description: Apache TsFile C++ library +Version: @TSFILE_PACKAGE_VERSION@ +URL: https://tsfile.apache.org/ +Libs: -L${libdir} -ltsfile +Cflags: -I${includedir} -I${includedir}/tsfile@TSFILE_PKGCONFIG_CFLAGS@ diff --git a/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt b/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt index 61901e38a..551373afe 100644 --- a/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt +++ b/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt @@ -18,7 +18,7 @@ under the License. ]] cmake_minimum_required(VERSION 3.11) -project(TsFileANTLR4DependencyTest NONE) +project(TsFileANTLR4DependencyTest LANGUAGES CXX) get_filename_component(_TSFILE_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE) diff --git a/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt new file mode 100644 index 000000000..bf0f83e34 --- /dev/null +++ b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt @@ -0,0 +1,42 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] +cmake_minimum_required(VERSION 3.11) +project(TsFileInstalledConsumer LANGUAGES CXX) + +find_package(TsFile CONFIG REQUIRED) + +get_target_property(_TSFILE_INCLUDE_DIRS TsFile::tsfile + INTERFACE_INCLUDE_DIRECTORIES) +if (NOT _TSFILE_INCLUDE_DIRS MATCHES "/include") + message(FATAL_ERROR "TsFile target does not expose the installed include root") +endif () +if (_TSFILE_INCLUDE_DIRS MATCHES "cpp/src|build/") + message(FATAL_ERROR "TsFile target leaks a source or build include path: ${_TSFILE_INCLUDE_DIRS}") +endif () + +get_target_property(_TSFILE_LINK_LIBRARIES TsFile::tsfile + INTERFACE_LINK_LIBRARIES) +if (_TSFILE_LINK_LIBRARIES MATCHES "(^|;)[^;]*_obj($|;)|cpp/src|build/") + message(FATAL_ERROR + "TsFile target leaks private build link requirements: " + "${_TSFILE_LINK_LIBRARIES}") +endif () + +add_executable(tsfile_installed_consumer main.cc) +target_link_libraries(tsfile_installed_consumer PRIVATE TsFile::tsfile) diff --git a/cpp/cmake/tests/projects/InstalledConsumer/main.cc b/cpp/cmake/tests/projects/InstalledConsumer/main.cc new file mode 100644 index 000000000..8d242f678 --- /dev/null +++ b/cpp/cmake/tests/projects/InstalledConsumer/main.cc @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include +#include + +int main() { return TS_DATATYPE_INT32 == 1 ? 0 : 1; } diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt b/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt new file mode 100644 index 000000000..1aaabc396 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt @@ -0,0 +1,32 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] +cmake_minimum_required(VERSION 3.11) +project(TsFilePkgConfigConsumer LANGUAGES C CXX) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(TsFile REQUIRED IMPORTED_TARGET libtsfile) +if (NOT TsFile_INCLUDEDIR MATCHES "/include$") + message(FATAL_ERROR "pkg-config did not report the installed include root: ${TsFile_INCLUDEDIR}") +endif () + +add_executable(tsfile_pkgconfig_consumer main.c) +target_link_libraries(tsfile_pkgconfig_consumer PRIVATE PkgConfig::TsFile) + +add_executable(tsfile_pkgconfig_cpp_consumer main.cc) +target_link_libraries(tsfile_pkgconfig_cpp_consumer PRIVATE PkgConfig::TsFile) diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.c b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c new file mode 100644 index 000000000..f32e5a550 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include + +int main(void) { return TS_DATATYPE_INT32 == 1 ? 0 : 1; } diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc new file mode 100644 index 000000000..be94b8543 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include + +int main() { return 0; } diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 2ab2dd5d6..8854e5d6c 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -79,6 +79,18 @@ add_subdirectory(reader) add_subdirectory(utils) add_subdirectory(writer) +# Object libraries need the directory-level project dependencies on older +# CMake versions, but those dependencies are implementation details of the +# shared library and must not become exported consumer requirements. Keep the +# platform thread target in the directory scope for its public link interface, +# then link codecs explicitly as PRIVATE below. +get_property(_TSFILE_SRC_LINK_LIBRARIES DIRECTORY PROPERTY LINK_LIBRARIES) +if (ENABLE_THREADS) + set_property(DIRECTORY PROPERTY LINK_LIBRARIES Threads::Threads) +else () + set_property(DIRECTORY PROPERTY LINK_LIBRARIES "") +endif () + set(_TSFILE_OBJECT_TARGETS common_obj compress_obj @@ -125,35 +137,115 @@ if (${COV_ENABLED}) else() set(COV_LINK_LIB -lgcov) endif() - if (CMAKE_VERSION VERSION_LESS "3.12") - target_link_libraries(tsfile ${COV_LINK_LIB}) - else() - target_link_libraries(tsfile ${_TSFILE_OBJECT_TARGETS} ${COV_LINK_LIB}) - endif() + target_link_libraries(tsfile PRIVATE ${COV_LINK_LIB}) else() message("Disable code cov...") - if (NOT CMAKE_VERSION VERSION_LESS "3.12") - target_link_libraries(tsfile ${_TSFILE_OBJECT_TARGETS}) - endif() endif() +# Linking object libraries directly makes static-library exports depend on +# private, non-exported object targets. Add their object files as sources +# instead; this works on the CMake 3.11 baseline and keeps the install target +# self-contained for both shared and static builds. +foreach (_TSFILE_OBJECT_TARGET IN LISTS _TSFILE_OBJECT_TARGETS) + target_sources(tsfile PRIVATE + $) +endforeach () + +if (NOT "${_TSFILE_PROJECT_DEPENDENCIES}" STREQUAL "") + target_link_libraries(tsfile PRIVATE ${_TSFILE_PROJECT_DEPENDENCIES}) +endif () + unset(_TSFILE_OBJECT_SOURCES) unset(_TSFILE_OBJECT_TARGET) unset(_TSFILE_OBJECT_TARGETS) add_dependencies(tsfile utils_obj encoding_obj) +set_property(DIRECTORY PROPERTY LINK_LIBRARIES + "${_TSFILE_SRC_LINK_LIBRARIES}") +unset(_TSFILE_SRC_LINK_LIBRARIES) + if (TSFILE_BUILD_SHARED) - set(LIBTSFILE_PROJECT_VERSION ${TsFile_CPP_VERSION}) - set(LIBTSFILE_SO_VERSION ${TsFile_CPP_VERSION}) + # Keep development suffixes out of the install name and SONAME. The ABI + # epoch changes only for incompatible binary interface changes. + set(LIBTSFILE_PROJECT_VERSION ${TSFILE_PACKAGE_VERSION}) + set(LIBTSFILE_SO_VERSION ${TSFILE_ABI_VERSION}) set_target_properties(tsfile PROPERTIES VERSION ${LIBTSFILE_PROJECT_VERSION}) set_target_properties(tsfile PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION}) endif() +add_library(TsFile::tsfile ALIAS tsfile) + +target_include_directories(tsfile PUBLIC + $ + $ + $ + $) +if (ENABLE_SIMD AND TSFILE_SIMDE_INCLUDE_ROOT) + # Public TsFile headers include SIMDe directly. Copy the resolved header + # tree into the install prefix so consumers do not need the build host's + # system or dependency-cache paths. + target_include_directories(tsfile PUBLIC + $ + $) +endif () +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_INCLUDE_ROOT) + target_include_directories(tsfile PUBLIC + $ + $) +endif () +if (ENABLE_ANTLR4 AND TSFILE_UTF8CPP_INCLUDE_ROOT) + target_include_directories(tsfile PUBLIC + $ + $) +endif () +target_compile_features(tsfile PUBLIC cxx_std_11) +set(_TSFILE_PUBLIC_FEATURE_DEFINITIONS "") +foreach (_TSFILE_PUBLIC_FEATURE + ENABLE_ANTLR4 + ENABLE_MEM_STAT + ENABLE_SNAPPY + ENABLE_LZ4 + ENABLE_LZOKAY + ENABLE_ZLIB + ENABLE_GZIP + ENABLE_ZSTD + ENABLE_LZMA2 + ENABLE_THREADS + ENABLE_SIMD) + if (${_TSFILE_PUBLIC_FEATURE}) + list(APPEND _TSFILE_PUBLIC_FEATURE_DEFINITIONS + ${_TSFILE_PUBLIC_FEATURE}) + endif () +endforeach () +if (NOT "${_TSFILE_PUBLIC_FEATURE_DEFINITIONS}" STREQUAL "") + target_compile_definitions(tsfile INTERFACE + ${_TSFILE_PUBLIC_FEATURE_DEFINITIONS}) +endif () +unset(_TSFILE_PUBLIC_FEATURE) +unset(_TSFILE_PUBLIC_FEATURE_DEFINITIONS) + +if (TSFILE_BUILD_SHARED) + # A shared library already resolves its private codec and object-library + # dependencies at link time. Do not export those build-only targets as + # consumer link requirements; they are not installed with the package. + set_property(TARGET tsfile PROPERTY INTERFACE_LINK_LIBRARIES "") +endif () + # A shared library is a RUNTIME plus an import ARCHIVE on Windows and a LIBRARY # on Unix. A static library is an ARCHIVE on every platform. Cover all three so # the install step works for either library type. install(TARGETS tsfile - RUNTIME DESTINATION ${LIBRARY_OUTPUT_PATH} - LIBRARY DESTINATION ${LIBRARY_OUTPUT_PATH} - ARCHIVE DESTINATION ${LIBRARY_OUTPUT_PATH}) + EXPORT TsFileTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT runtime + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT runtime + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT development) + +install(EXPORT TsFileTargets + FILE TsFileTargets.cmake + NAMESPACE TsFile:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/TsFile + COMPONENT development) diff --git a/cpp/third_party/CMakeLists.txt b/cpp/third_party/CMakeLists.txt index f7a8851b5..203b93c34 100755 --- a/cpp/third_party/CMakeLists.txt +++ b/cpp/third_party/CMakeLists.txt @@ -35,6 +35,12 @@ if (ENABLE_ANTLR4) "${TSFILE_UTF8CPP_SOURCE_DIR}/source") add_library(tsfile_antlr4_bundled STATIC ${_TSFILE_ANTLR4_SOURCES}) + set(TSFILE_ANTLR4_INCLUDE_ROOT + "${TSFILE_ANTLR4_SOURCE_DIR}/runtime/Cpp/runtime/src" + CACHE INTERNAL "Resolved ANTLR4 include root" FORCE) + set(TSFILE_UTF8CPP_INCLUDE_ROOT + "${TSFILE_UTF8CPP_SOURCE_DIR}/source" + CACHE INTERNAL "Resolved utf8cpp include root" FORCE) set_target_properties(tsfile_antlr4_bundled PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_STANDARD 11) @@ -93,6 +99,17 @@ if (ENABLE_ANTLR4) target_link_libraries(tsfile_antlr4 INTERFACE tsfile_antlr4_bundled) else () + set(TSFILE_ANTLR4_INCLUDE_ROOT + "${TSFILE_ANTLR4_SYSTEM_INCLUDE_DIR}" + CACHE INTERNAL "Resolved ANTLR4 include root" FORCE) + if (TARGET utf8cpp) + get_target_property(_TSFILE_UTF8CPP_INCLUDE_ROOT utf8cpp + INTERFACE_INCLUDE_DIRECTORIES) + set(TSFILE_UTF8CPP_INCLUDE_ROOT + "${_TSFILE_UTF8CPP_INCLUDE_ROOT}" + CACHE INTERNAL "Resolved utf8cpp include root" FORCE) + unset(_TSFILE_UTF8CPP_INCLUDE_ROOT) + endif () target_link_libraries(tsfile_antlr4 INTERFACE ${TSFILE_ANTLR4_SYSTEM_TARGET}) if (TSFILE_ANTLR4_SYSTEM_INCLUDE_DIR) @@ -449,9 +466,15 @@ if (ENABLE_SIMD) add_library(tsfile_simde INTERFACE) if (TSFILE_SIMDE_SOURCE STREQUAL "BUNDLED") + set(TSFILE_SIMDE_INCLUDE_ROOT "${TSFILE_SIMDE_SOURCE_DIR}" + CACHE INTERNAL "Resolved SIMDe include root" FORCE) target_include_directories(tsfile_simde INTERFACE "${TSFILE_SIMDE_SOURCE_DIR}") else () + get_target_property(TSFILE_SIMDE_INCLUDE_ROOT simde::simde + INTERFACE_INCLUDE_DIRECTORIES) + set(TSFILE_SIMDE_INCLUDE_ROOT "${TSFILE_SIMDE_INCLUDE_ROOT}" + CACHE INTERNAL "Resolved SIMDe include root" FORCE) target_link_libraries(tsfile_simde INTERFACE simde::simde) endif () add_library(TsFile::SIMDe ALIAS tsfile_simde) diff --git a/cpp/tools/CMakeLists.txt b/cpp/tools/CMakeLists.txt index 00bc20ea0..b78d06442 100644 --- a/cpp/tools/CMakeLists.txt +++ b/cpp/tools/CMakeLists.txt @@ -59,4 +59,19 @@ set_target_properties(tsfile_cli PROPERTIES OUTPUT_NAME tsfile-cli RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin) -install(TARGETS tsfile_cli RUNTIME DESTINATION bin) +# Keep the installed CLI self-contained with the sibling lib directory. This +# is relative to the executable location and therefore remains valid when the +# installation prefix is relocated. +if (APPLE) + set_target_properties(tsfile_cli PROPERTIES + INSTALL_RPATH "@loader_path/../${CMAKE_INSTALL_LIBDIR}" + BUILD_WITH_INSTALL_RPATH TRUE) +elseif (UNIX) + set_target_properties(tsfile_cli PROPERTIES + INSTALL_RPATH "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}" + BUILD_WITH_INSTALL_RPATH TRUE) +endif () + +install(TARGETS tsfile_cli + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT tools) diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 000000000..ff77cf46b --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,92 @@ + + +# TsFile C++ Packages + +The C++ installation contract is shared by all package formats. It installs +the library, the current compatibility header closure, `TsFileConfig.cmake`, +pkg-config metadata, the CLI, and Apache license files below one prefix. + +The header closure mirrors the include relationships used by the current C++ +implementation. It is intentionally broader than the long-term stable public +API; a separate API cleanup will narrow it in a future version. + +## Portable archive + +Build a relocatable source archive on any host with CMake and CPack: + +```bash +cmake -S cpp -B cpp/build/package \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO +cmake --build cpp/build/package --parallel +cmake --install cpp/build/package +cpack --config cpp/build/package/CPackConfig.cmake -G TGZ +``` + +The resulting `tsfile--.tar.gz` contains a standard +prefix layout and can be unpacked at `/usr/local`, a user directory, or a +relocated application prefix. + +## Native Linux packages + +DEB and RPM packages must be built in the target distribution environment so +CPack can run the native dependency scanner (`dpkg-shlibdeps` or +`rpmbuild`). On Debian/Ubuntu use `-G DEB`; on Fedora/RHEL use `-G RPM`: + +```bash +cmake -S cpp -B cpp/build/package \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO +cmake --build cpp/build/package --parallel +cmake --install cpp/build/package +cpack --config cpp/build/package/CPackConfig.cmake -G DEB +# or: cpack --config cpp/build/package/CPackConfig.cmake -G RPM +``` + +`SYSTEM` can be used instead of `AUTO` when the build image provides every +compatible dependency, including ANTLR4 4.9.x. `AUTO` is the reproducible +release default and uses the verified source fallback for unavailable or +incompatible distro versions. + +The repository workflow `Cpp-Packaging` builds and uploads native DEB and RPM +artifacts on packaging-related changes. The DEB job runs on Ubuntu and the RPM +job runs in Fedora, so each package is generated with its native dependency +metadata tool. + +## macOS + +Homebrew is the native macOS distribution path. The formula is +`packaging/homebrew/tsfile.rb`; it builds the same CMake install layout and +tests both a C++ consumer and `tsfile-cli --version` after installation. + +```bash +brew install ./packaging/homebrew/tsfile.rb +``` + +The stable formula URL and checksum must be updated to the ASF source archive +when the first release containing this packaging work is published. diff --git a/packaging/homebrew/tsfile.rb b/packaging/homebrew/tsfile.rb new file mode 100644 index 000000000..3e96f3dca --- /dev/null +++ b/packaging/homebrew/tsfile.rb @@ -0,0 +1,71 @@ +# typed: strict +# frozen_string_literal: true + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Formula for the Apache TsFile C++ library. +class Tsfile < Formula + desc "Columnar storage library for time series data" + homepage "https://tsfile.apache.org/" + # Replace this development snapshot with the ASF source archive and its + # release checksum when the first native TsFile release is published. + url "https://github.com/apache/tsfile/archive/d4c3c94690cf9af819bc26bec7114a0de7900359.tar.gz" + version "2.3.2" + sha256 "369cf43742601fe6107b299647d2c01f705bfc9ff111402d7b5ddb9b50f40f16" + license "Apache-2.0" + head "https://github.com/apache/tsfile.git", branch: "develop" + + depends_on "cmake" => :build + depends_on "lz4" + depends_on "simde" + depends_on "snappy" + depends_on "utf8cpp" + depends_on "zstd" + + uses_from_macos "zlib" + + def install + args = %W[ + -DBUILD_TEST=OFF + -DBUILD_TOOLS=ON + -DENABLE_LZMA2=OFF + -DTSFILE_DEPENDENCY_SOURCE=AUTO + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + -DCMAKE_INSTALL_RPATH=#{rpath} + ] + + system "cmake", "-S", "cpp", "-B", "build", *args, *std_cmake_args + system "cmake", "--build", "build" + system "cmake", "--install", "build" + end + + test do + (testpath / "test.cpp").write <<~CPP + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + CPP + + system ENV.cxx, "-std=c++11", "test.cpp", "-I#{include}", "-L#{lib}", + "-Wl,-rpath,#{lib}", "-ltsfile", "-o", "test" + system "./test" + system bin / "tsfile-cli", "--version" + end +end From 3e4220f1b6894d0f1dc039cd35a0565d3b5bb403 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 17:14:33 +0800 Subject: [PATCH 02/27] fix(ci): remove redundant install step before cpack --- .github/workflows/cpp-packaging.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index ef5a457c2..197cd7b18 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -68,7 +68,6 @@ jobs: -DTSFILE_DEPENDENCY_SOURCE=AUTO \ -DTSFILE_ENABLE_NATIVE_ARCH=OFF cmake --build build/deb --parallel - cmake --install build/deb - name: Build and inspect DEB packages run: | @@ -111,7 +110,6 @@ jobs: -DTSFILE_DEPENDENCY_SOURCE=AUTO \ -DTSFILE_ENABLE_NATIVE_ARCH=OFF cmake --build build/rpm --parallel - cmake --install build/rpm - name: Build and inspect RPM packages run: | From 5fccdc4ddf5577716adc25d840fe69ada5c0a841 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 17:31:10 +0800 Subject: [PATCH 03/27] fix(ci): install uuid headers for bundled ANTLR4 --- .github/workflows/cpp-packaging.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 197cd7b18..54fa92211 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -56,7 +56,7 @@ jobs: - name: Install packaging tools run: | sudo apt-get update - sudo apt-get install -y build-essential cmake ninja-build pkg-config dpkg-dev + sudo apt-get install -y build-essential cmake ninja-build pkg-config dpkg-dev uuid-dev - name: Configure and build run: | @@ -95,7 +95,7 @@ jobs: run: | dnf install -y \ cmake gcc-c++ make ninja-build pkgconf-pkg-config rpm-build \ - git curl tar xz unzip gzip + git curl tar xz unzip gzip libuuid-devel - name: Checkout repository uses: actions/checkout@v7 From 4f5ed026c0a0b36702c2055fa6aa78fcb835bec8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:09:03 +0800 Subject: [PATCH 04/27] feat(ci): add native package development versions --- cpp/CMakeLists.txt | 23 ++- packaging/scripts/native_package_versions.py | 128 ++++++++++++++++ .../tests/test_native_package_versions.py | 144 ++++++++++++++++++ 3 files changed, 289 insertions(+), 6 deletions(-) create mode 100644 packaging/scripts/native_package_versions.py create mode 100644 packaging/tests/test_native_package_versions.py diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 567286fa5..4b6f868b2 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -47,6 +47,14 @@ if ("${TSFILE_PACKAGE_VERSION}" STREQUAL "") "TsFile_CPP_VERSION must start with a semantic version: " "${TsFile_CPP_VERSION}") endif () +set(TSFILE_ARCHIVE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "Version used in portable archive names") +set(TSFILE_DEBIAN_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "Complete Debian package version") +set(TSFILE_RPM_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "RPM Version value") +set(TSFILE_RPM_PACKAGE_RELEASE "1" CACHE STRING + "RPM Release value") set(TSFILE_ABI_VERSION "1" CACHE STRING "TsFile shared-library ABI epoch (changes only on ABI breaks)") if (NOT TSFILE_ABI_VERSION MATCHES "^[0-9]+$") @@ -547,7 +555,7 @@ if (TSFILE_ENABLE_CPACK) set(CPACK_PACKAGE_HOMEPAGE_URL "https://tsfile.apache.org/") set(CPACK_PACKAGE_CONTACT "dev@tsfile.apache.org") set(CPACK_RESOURCE_FILE_LICENSE "${TSFILE_LICENSE_FILE}") - set(CPACK_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}") + set(CPACK_PACKAGE_VERSION "${TSFILE_ARCHIVE_VERSION}") string(REPLACE "." ";" _TSFILE_PACKAGE_VERSION_PARTS "${TSFILE_PACKAGE_VERSION}") list(GET _TSFILE_PACKAGE_VERSION_PARTS 0 CPACK_PACKAGE_VERSION_MAJOR) @@ -560,25 +568,28 @@ if (TSFILE_ENABLE_CPACK) set(CPACK_DEB_COMPONENT_INSTALL ON) set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) + set(CPACK_DEBIAN_PACKAGE_VERSION "${TSFILE_DEBIAN_PACKAGE_VERSION}") set(CPACK_DEBIAN_RUNTIME_PACKAGE_NAME "tsfile") set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_NAME "tsfile-dev") set(CPACK_DEBIAN_TOOLS_PACKAGE_NAME "tsfile-tools") set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_DEPENDS - "tsfile (= ${TSFILE_PACKAGE_VERSION})") + "tsfile (= ${TSFILE_DEBIAN_PACKAGE_VERSION})") set(CPACK_DEBIAN_TOOLS_PACKAGE_DEPENDS - "tsfile (= ${TSFILE_PACKAGE_VERSION})") + "tsfile (= ${TSFILE_DEBIAN_PACKAGE_VERSION})") set(CPACK_RPM_COMPONENT_INSTALL ON) + set(CPACK_RPM_FILE_NAME RPM-DEFAULT) set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") - set(CPACK_RPM_PACKAGE_RELEASE "1") + set(CPACK_RPM_PACKAGE_VERSION "${TSFILE_RPM_PACKAGE_VERSION}") + set(CPACK_RPM_PACKAGE_RELEASE "${TSFILE_RPM_PACKAGE_RELEASE}") set(CPACK_RPM_PACKAGE_AUTOREQPROV ON) set(CPACK_RPM_RUNTIME_PACKAGE_NAME "tsfile") set(CPACK_RPM_DEVELOPMENT_PACKAGE_NAME "tsfile-devel") set(CPACK_RPM_TOOLS_PACKAGE_NAME "tsfile-tools") set(CPACK_RPM_DEVELOPMENT_PACKAGE_REQUIRES - "tsfile = ${TSFILE_PACKAGE_VERSION}-1") + "tsfile = ${TSFILE_RPM_PACKAGE_VERSION}-${TSFILE_RPM_PACKAGE_RELEASE}") set(CPACK_RPM_TOOLS_PACKAGE_REQUIRES - "tsfile = ${TSFILE_PACKAGE_VERSION}-1") + "tsfile = ${TSFILE_RPM_PACKAGE_VERSION}-${TSFILE_RPM_PACKAGE_RELEASE}") include(CPack) unset(_TSFILE_PACKAGE_VERSION_PARTS) diff --git a/packaging/scripts/native_package_versions.py b/packaging/scripts/native_package_versions.py new file mode 100644 index 000000000..f9849267f --- /dev/null +++ b/packaging/scripts/native_package_versions.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Build deterministic development versions for native TsFile packages.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Sequence + + +_SOURCE_VERSION_RE = re.compile( + r"^(?P(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))\.dev$" +) +_CMAKE_VERSION_RE = re.compile( + r"^\s*set\(\s*TsFile_CPP_VERSION\s+(?P[^\s)]+)\s*\)\s*(?:#.*)?$", + re.MULTILINE, +) +_BUILD_DATE_RE = re.compile(r"^[0-9]{8}$") +_GIT_SHA_RE = re.compile(r"^[0-9a-fA-F]{7,}$") + + +def read_cpp_version(path: Path) -> str: + """Read and validate the development version declared by CMake.""" + match = _CMAKE_VERSION_RE.search(path.read_text(encoding="utf-8")) + if match is None or _SOURCE_VERSION_RE.fullmatch(match.group("version")) is None: + raise ValueError(f"{path} must declare TsFile_CPP_VERSION as MAJOR.MINOR.PATCH.dev") + return match.group("version") + + +def _positive_integer(value: int, name: str) -> str: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{name} must be a positive integer") + return str(value) + + +def build_versions( + source_version: str, + build_date: str, + run_number: int, + run_attempt: int, + git_sha: str, +) -> dict[str, str]: + """Create archive, DEB, RPM, and Homebrew versions from one build identity.""" + source_match = _SOURCE_VERSION_RE.fullmatch(source_version) + if source_match is None: + raise ValueError("source_version must be MAJOR.MINOR.PATCH.dev") + if _BUILD_DATE_RE.fullmatch(build_date) is None: + raise ValueError("build_date must be YYYYMMDD") + if _GIT_SHA_RE.fullmatch(git_sha) is None: + raise ValueError("git_sha must contain at least seven hexadecimal characters") + + run_number_text = _positive_integer(run_number, "run_number") + run_attempt_text = _positive_integer(run_attempt, "run_attempt") + base_version = source_match.group("base") + short_sha = git_sha[:7].lower() + build_identity = f"{build_date}.{run_number_text}.{run_attempt_text}.g{short_sha}" + + return { + "base_version": base_version, + "logical_version": f"{base_version}.dev0+{build_identity}", + "deb_version": f"{base_version}~dev0+{build_identity}-1", + "rpm_version": base_version, + "rpm_release": f"0.dev0.{build_identity}.el9", + "archive_version": f"{base_version}-dev0.{build_identity}", + "homebrew_version": f"{base_version}.dev0.{build_identity}", + } + + +def _parse_positive_integer(value: str) -> int: + if not re.fullmatch(r"[1-9][0-9]*", value): + raise argparse.ArgumentTypeError("must be a positive integer") + return int(value) + + +def _argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate deterministic development versions for native TsFile packages." + ) + parser.add_argument("--cmake-file", type=Path, required=True) + parser.add_argument("--build-date", required=True) + parser.add_argument("--run-number", type=_parse_positive_integer, required=True) + parser.add_argument("--run-attempt", type=_parse_positive_integer, required=True) + parser.add_argument("--git-sha", required=True) + parser.add_argument("--json-out", type=Path, required=True) + parser.add_argument("--github-output", type=Path) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Write the generated versions to JSON and, optionally, GitHub outputs.""" + arguments = _argument_parser().parse_args(argv) + versions = build_versions( + read_cpp_version(arguments.cmake_file), + arguments.build_date, + arguments.run_number, + arguments.run_attempt, + arguments.git_sha, + ) + arguments.json_out.write_text(json.dumps(versions, indent=2) + "\n", encoding="utf-8") + if arguments.github_output is not None: + arguments.github_output.write_text( + "\n".join(f"{key}={value}" for key, value in versions.items()) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py new file mode 100644 index 000000000..6b864c44c --- /dev/null +++ b/packaging/tests/test_native_package_versions.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = REPOSITORY_ROOT / "packaging/scripts/native_package_versions.py" +CPP_CMAKE_FILE = REPOSITORY_ROOT / "cpp/CMakeLists.txt" + + +def load_version_module(): + if not MODULE_PATH.is_file(): + return None + spec = importlib.util.spec_from_file_location("native_package_versions", MODULE_PATH) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load native package version helper from {MODULE_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class NativePackageVersionsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = load_version_module() + + def require_module(self): + self.assertIsNotNone( + self.module, "native_package_versions helper must exist for native package builds" + ) + return self.module + + def test_reads_development_version_from_cpp_cmake(self): + module = self.require_module() + + self.assertEqual(module.read_cpp_version(CPP_CMAKE_FILE), "2.5.0.dev") + + def test_builds_exact_development_package_versions(self): + module = self.require_module() + + self.assertEqual( + module.build_versions("2.5.0.dev", "20260910", 123, 1, "abcdef123456"), + { + "base_version": "2.5.0", + "logical_version": "2.5.0.dev0+20260910.123.1.gabcdef1", + "deb_version": "2.5.0~dev0+20260910.123.1.gabcdef1-1", + "rpm_version": "2.5.0", + "rpm_release": "0.dev0.20260910.123.1.gabcdef1.el9", + "archive_version": "2.5.0-dev0.20260910.123.1.gabcdef1", + "homebrew_version": "2.5.0.dev0.20260910.123.1.gabcdef1", + }, + ) + + def test_rejects_invalid_source_versions(self): + module = self.require_module() + + for source_version in ("2.5.0", "2.5.dev"): + with self.subTest(source_version=source_version): + with self.assertRaises(ValueError): + module.build_versions(source_version, "20260910", 123, 1, "abcdef123456") + + def test_rejects_invalid_build_identity_fields(self): + module = self.require_module() + + invalid_cases = ( + ("2026-09-10", 123, 1, "abcdef123456"), + ("2026091", 123, 1, "abcdef123456"), + ("20260910", "run", 1, "abcdef123456"), + ("20260910", 123, "attempt", "abcdef123456"), + ("20260910", 123, 1, "abcdef"), + ("20260910", 123, 1, "abcdefg123456"), + ) + for build_date, run_number, run_attempt, git_sha in invalid_cases: + with self.subTest( + build_date=build_date, + run_number=run_number, + run_attempt=run_attempt, + git_sha=git_sha, + ): + with self.assertRaises(ValueError): + module.build_versions( + "2.5.0.dev", build_date, run_number, run_attempt, git_sha + ) + + def test_cli_writes_json_and_github_outputs(self): + module = self.require_module() + expected = module.build_versions("2.5.0.dev", "20260910", 123, 1, "abcdef123456") + + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_path = Path(temporary_directory) + json_output = temporary_path / "versions.json" + github_output = temporary_path / "github-output.txt" + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--cmake-file", + str(CPP_CMAKE_FILE), + "--build-date", + "20260910", + "--run-number", + "123", + "--run-attempt", + "1", + "--git-sha", + "abcdef123456", + "--json-out", + str(json_output), + "--github-output", + str(github_output), + ], + check=True, + ) + + self.assertEqual(json.loads(json_output.read_text(encoding="utf-8")), expected) + self.assertEqual( + github_output.read_text(encoding="utf-8").splitlines(), + [f"{key}={value}" for key, value in expected.items()], + ) + + +if __name__ == "__main__": + unittest.main() From 15e69f2879e03235cfbdf1b92990ce9dc1d29497 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:16:10 +0800 Subject: [PATCH 05/27] fix(ci): validate native package build identity --- packaging/scripts/native_package_versions.py | 10 ++++++---- packaging/tests/test_native_package_versions.py | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packaging/scripts/native_package_versions.py b/packaging/scripts/native_package_versions.py index f9849267f..d4bd653eb 100644 --- a/packaging/scripts/native_package_versions.py +++ b/packaging/scripts/native_package_versions.py @@ -23,6 +23,7 @@ import argparse import json import re +from datetime import datetime from pathlib import Path from typing import Sequence @@ -65,6 +66,7 @@ def build_versions( raise ValueError("source_version must be MAJOR.MINOR.PATCH.dev") if _BUILD_DATE_RE.fullmatch(build_date) is None: raise ValueError("build_date must be YYYYMMDD") + datetime.strptime(build_date, "%Y%m%d") if _GIT_SHA_RE.fullmatch(git_sha) is None: raise ValueError("git_sha must contain at least seven hexadecimal characters") @@ -117,10 +119,10 @@ def main(argv: Sequence[str] | None = None) -> int: ) arguments.json_out.write_text(json.dumps(versions, indent=2) + "\n", encoding="utf-8") if arguments.github_output is not None: - arguments.github_output.write_text( - "\n".join(f"{key}={value}" for key, value in versions.items()) + "\n", - encoding="utf-8", - ) + with arguments.github_output.open("a", encoding="utf-8") as github_output: + github_output.write( + "\n".join(f"{key}={value}" for key, value in versions.items()) + "\n" + ) return 0 diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py index 6b864c44c..a2cc05221 100644 --- a/packaging/tests/test_native_package_versions.py +++ b/packaging/tests/test_native_package_versions.py @@ -86,6 +86,7 @@ def test_rejects_invalid_build_identity_fields(self): invalid_cases = ( ("2026-09-10", 123, 1, "abcdef123456"), ("2026091", 123, 1, "abcdef123456"), + ("20261340", 123, 1, "abcdef123456"), ("20260910", "run", 1, "abcdef123456"), ("20260910", 123, "attempt", "abcdef123456"), ("20260910", 123, 1, "abcdef"), @@ -111,6 +112,7 @@ def test_cli_writes_json_and_github_outputs(self): temporary_path = Path(temporary_directory) json_output = temporary_path / "versions.json" github_output = temporary_path / "github-output.txt" + github_output.write_text("existing=value\n", encoding="utf-8") subprocess.run( [ sys.executable, @@ -136,7 +138,7 @@ def test_cli_writes_json_and_github_outputs(self): self.assertEqual(json.loads(json_output.read_text(encoding="utf-8")), expected) self.assertEqual( github_output.read_text(encoding="utf-8").splitlines(), - [f"{key}={value}" for key, value in expected.items()], + ["existing=value", *[f"{key}={value}" for key, value in expected.items()]], ) From b5a8b30c8ed007f468dddac7e68884cba2b8262b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:26:29 +0800 Subject: [PATCH 06/27] feat(ci): build native Linux package artifacts --- .github/workflows/native-packages.yml | 328 ++++++++++++++++++++++++++ packaging/README.md | 11 +- 2 files changed, 335 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/native-packages.yml diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml new file mode 100644 index 000000000..5c1bafa5e --- /dev/null +++ b/.github/workflows/native-packages.yml @@ -0,0 +1,328 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build native package artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + prepare: + name: Prepare package versions + runs-on: ubuntu-24.04 + outputs: + base_version: ${{ steps.versions.outputs.base_version }} + logical_version: ${{ steps.versions.outputs.logical_version }} + deb_version: ${{ steps.versions.outputs.deb_version }} + rpm_version: ${{ steps.versions.outputs.rpm_version }} + rpm_release: ${{ steps.versions.outputs.rpm_release }} + archive_version: ${{ steps.versions.outputs.archive_version }} + homebrew_version: ${{ steps.versions.outputs.homebrew_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Generate package versions + id: versions + run: | + python3 packaging/scripts/native_package_versions.py \ + --cmake-file cpp/CMakeLists.txt \ + --build-date "$(date -u +%Y%m%d)" \ + --run-number "${{ github.run_number }}" \ + --run-attempt "${{ github.run_attempt }}" \ + --git-sha "${{ github.sha }}" \ + --json-out versions.json \ + --github-output "$GITHUB_OUTPUT" + + - name: Upload package versions + uses: actions/upload-artifact@v7 + with: + name: native-package-versions + path: versions.json + if-no-files-found: error + retention-days: 14 + + build-deb: + name: Build Ubuntu 22.04 DEB packages + needs: prepare + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install DEB build prerequisites + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + cmake \ + ninja-build \ + pkg-config \ + dpkg-dev \ + uuid-dev + + - name: Configure and build + env: + TSFILE_ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + TSFILE_DEBIAN_PACKAGE_VERSION: ${{ needs.prepare.outputs.deb_version }} + run: | + cmake -S cpp -B build/deb -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO \ + -DTSFILE_ENABLE_NATIVE_ARCH=OFF \ + -DTSFILE_ARCHIVE_VERSION="$TSFILE_ARCHIVE_VERSION" \ + -DTSFILE_DEBIAN_PACKAGE_VERSION="$TSFILE_DEBIAN_PACKAGE_VERSION" + cmake --build build/deb --parallel + + - name: Build and verify DEB packages + env: + EXPECTED_DEB_VERSION: ${{ needs.prepare.outputs.deb_version }} + shell: bash + run: | + mkdir -p packages + cpack -G DEB --config build/deb/CPackConfig.cmake -B packages + + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.deb' -print | sort) + if [[ ${#packages[@]} -ne 3 ]]; then + printf 'Expected exactly 3 DEB packages, found %s\n' "${#packages[@]}" + exit 1 + fi + + actual_names="$({ + for package in "${packages[@]}"; do + dpkg-deb --field "$package" Package + done + } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-dev tsfile-tools | sort)" + if [[ "$actual_names" != "$expected_names" ]]; then + printf 'Unexpected DEB package names:\n%s\n' "$actual_names" + exit 1 + fi + + for package in "${packages[@]}"; do + package_version="$(dpkg-deb --field "$package" Version)" + if [[ "$package_version" != "$EXPECTED_DEB_VERSION" ]]; then + printf '%s has version %s, expected %s\n' \ + "$package" "$package_version" "$EXPECTED_DEB_VERSION" + exit 1 + fi + done + + - name: Upload DEB packages + uses: actions/upload-artifact@v7 + with: + name: native-deb-ubuntu22.04-amd64 + path: packages/*.deb + if-no-files-found: error + retention-days: 14 + + test-deb: + name: Test DEB packages on ${{ matrix.container }} + needs: build-deb + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + container: + - ubuntu:22.04 + - ubuntu:24.04 + container: ${{ matrix.container }} + steps: + - name: Download DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-amd64 + path: packages + + - name: Install DEB packages + shell: bash + run: | + apt-get update + apt-get install -y ./packages/*.deb cmake build-essential + + - name: Verify installed DEB packages + run: | + tsfile-cli --version + dpkg-query -W tsfile tsfile-dev tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + mkdir -p consumer + cat > consumer/CMakeLists.txt <<'CMAKE' + cmake_minimum_required(VERSION 3.11) + project(TsFileInstalledConsumer LANGUAGES CXX) + + find_package(TsFile CONFIG REQUIRED) + add_executable(installed_consumer main.cpp) + target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) + CMAKE + cat > consumer/main.cpp <<'CPP' + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + CPP + cmake -S consumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/installed_consumer + + build-rpm: + name: Build AlmaLinux 9 RPM packages + needs: prepare + runs-on: ubuntu-24.04 + container: almalinux:9 + timeout-minutes: 30 + steps: + - name: Install RPM build prerequisites + run: | + dnf install -y \ + cmake \ + gcc-c++ \ + ninja-build \ + pkgconf-pkg-config \ + rpm-build \ + git \ + curl \ + tar \ + gzip \ + libuuid-devel + + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Configure and build + env: + TSFILE_ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + TSFILE_RPM_PACKAGE_VERSION: ${{ needs.prepare.outputs.rpm_version }} + TSFILE_RPM_PACKAGE_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + cmake -S cpp -B build/rpm -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO \ + -DTSFILE_ENABLE_NATIVE_ARCH=OFF \ + -DTSFILE_ARCHIVE_VERSION="$TSFILE_ARCHIVE_VERSION" \ + -DTSFILE_RPM_PACKAGE_VERSION="$TSFILE_RPM_PACKAGE_VERSION" \ + -DTSFILE_RPM_PACKAGE_RELEASE="$TSFILE_RPM_PACKAGE_RELEASE" + cmake --build build/rpm --parallel + + - name: Build and verify RPM packages + env: + EXPECTED_RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + EXPECTED_RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + mkdir -p packages + cpack -G RPM --config build/rpm/CPackConfig.cmake -B packages + + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.rpm' -print | sort) + if [[ ${#packages[@]} -ne 3 ]]; then + printf 'Expected exactly 3 RPM packages, found %s\n' "${#packages[@]}" + exit 1 + fi + + actual_names="$({ + for package in "${packages[@]}"; do + rpm -qp --queryformat '%{NAME}\n' "$package" + done + } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-devel tsfile-tools | sort)" + if [[ "$actual_names" != "$expected_names" ]]; then + printf 'Unexpected RPM package names:\n%s\n' "$actual_names" + exit 1 + fi + + for package in "${packages[@]}"; do + package_version="$(rpm -qp --queryformat '%{VERSION}' "$package")" + package_release="$(rpm -qp --queryformat '%{RELEASE}' "$package")" + if [[ "$package_version" != "$EXPECTED_RPM_VERSION" ]]; then + printf '%s has version %s, expected %s\n' \ + "$package" "$package_version" "$EXPECTED_RPM_VERSION" + exit 1 + fi + if [[ "$package_release" != "$EXPECTED_RPM_RELEASE" ]]; then + printf '%s has release %s, expected %s\n' \ + "$package" "$package_release" "$EXPECTED_RPM_RELEASE" + exit 1 + fi + done + + - name: Upload RPM packages + uses: actions/upload-artifact@v7 + with: + name: native-rpm-almalinux9-x86_64 + path: packages/*.rpm + if-no-files-found: error + retention-days: 14 + + test-rpm: + name: Test RPM packages on AlmaLinux 9 + needs: build-rpm + runs-on: ubuntu-24.04 + container: almalinux:9 + steps: + - name: Download RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-x86_64 + path: packages + + - name: Install RPM packages + run: dnf install -y packages/*.rpm cmake gcc-c++ + + - name: Verify installed RPM packages + run: | + tsfile-cli --version + rpm -q tsfile tsfile-devel tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + mkdir -p consumer + cat > consumer/CMakeLists.txt <<'CMAKE' + cmake_minimum_required(VERSION 3.11) + project(TsFileInstalledConsumer LANGUAGES CXX) + + find_package(TsFile CONFIG REQUIRED) + add_executable(installed_consumer main.cpp) + target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) + CMAKE + cat > consumer/main.cpp <<'CPP' + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + CPP + cmake -S consumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/installed_consumer diff --git a/packaging/README.md b/packaging/README.md index ff77cf46b..5cb218c78 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -73,10 +73,13 @@ compatible dependency, including ANTLR4 4.9.x. `AUTO` is the reproducible release default and uses the verified source fallback for unavailable or incompatible distro versions. -The repository workflow `Cpp-Packaging` builds and uploads native DEB and RPM -artifacts on packaging-related changes. The DEB job runs on Ubuntu and the RPM -job runs in Fedora, so each package is generated with its native dependency -metadata tool. +The manual `Build native package artifacts` workflow is the authoritative +native package build. Its Linux jobs create `tsfile`, `tsfile-dev`, and +`tsfile-tools` DEBs on Ubuntu 22.04 and `tsfile`, `tsfile-devel`, and +`tsfile-tools` RPMs on AlmaLinux 9. Fresh Ubuntu 22.04, Ubuntu 24.04, and +AlmaLinux 9 containers install the packages and verify both `tsfile-cli` and an +external CMake consumer. The workflow uploads intermediate artifacts for 14 +days and does not publish packages. ## macOS From 0958b56fed6e7941bd145f0d8fee4c623d552d2d Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:33:54 +0800 Subject: [PATCH 07/27] fix(ci): verify native CLI package ownership --- .github/workflows/native-packages.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 5c1bafa5e..79adbb5c3 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -165,7 +165,16 @@ jobs: apt-get install -y ./packages/*.deb cmake build-essential - name: Verify installed DEB packages + shell: bash run: | + cli_path="$(command -v tsfile-cli)" + cli_path="$(readlink -f -- "$cli_path")" + cli_owner="$(dpkg-query -S "$cli_path")" + cli_owner="${cli_owner%%:*}" + if [[ "$cli_owner" != "tsfile-tools" ]]; then + printf '%s is owned by %s, expected tsfile-tools\n' "$cli_path" "$cli_owner" + exit 1 + fi tsfile-cli --version dpkg-query -W tsfile tsfile-dev tsfile-tools @@ -300,7 +309,15 @@ jobs: run: dnf install -y packages/*.rpm cmake gcc-c++ - name: Verify installed RPM packages + shell: bash run: | + cli_path="$(command -v tsfile-cli)" + cli_path="$(readlink -f -- "$cli_path")" + cli_owner="$(rpm -qf --queryformat '%{NAME}' "$cli_path")" + if [[ "$cli_owner" != "tsfile-tools" ]]; then + printf '%s is owned by %s, expected tsfile-tools\n' "$cli_path" "$cli_owner" + exit 1 + fi tsfile-cli --version rpm -q tsfile tsfile-devel tsfile-tools From 8519ed5b8fe4001784268837fd73883194a85aa5 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:57:43 +0800 Subject: [PATCH 08/27] feat(ci): build Windows native SDK archive --- .github/workflows/native-packages.yml | 133 ++++++++++++++++++++++++++ cpp/CMakeLists.txt | 9 +- packaging/README.md | 16 ++++ 3 files changed, 157 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 79adbb5c3..8b6bd3e4e 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -343,3 +343,136 @@ jobs: cmake -S consumer -B consumer/build cmake --build consumer/build --parallel ./consumer/build/installed_consumer + + build-windows: + name: Build Windows MSVC x86_64 ZIP + needs: prepare + runs-on: windows-2022 + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Configure the MSVC environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Configure, build, and stage + env: + TSFILE_ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + cmake -S cpp -B build/windows -G "Visual Studio 17 2022" -A x64 ` + -DBUILD_TEST=OFF ` + -DBUILD_TOOLS=ON ` + -DTSFILE_ENABLE_CPACK=ON ` + -DTSFILE_DEPENDENCY_SOURCE=BUNDLED ` + -DTSFILE_ENABLE_NATIVE_ARCH=OFF ` + -DTSFILE_ARCHIVE_VERSION="$env:TSFILE_ARCHIVE_VERSION" + cmake --build build/windows --config Release --parallel + cmake --install build/windows --config Release --prefix stage/windows + + - name: Test staged CLI and SDK + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $stage = (Resolve-Path stage/windows).Path + $requiredPaths = @( + "$stage/bin/tsfile-cli.exe" + "$stage/bin/tsfile.dll" + "$stage/lib/tsfile.lib" + "$stage/include/tsfile/cwrapper/tsfile_cwrapper.h" + "$stage/lib/cmake/TsFile/TsFileConfig.cmake" + "$stage/share/doc/tsfile/LICENSE" + "$stage/share/doc/tsfile/NOTICE" + ) + foreach ($path in $requiredPaths) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Required staged file is absent: $path" + } + } + + $env:PATH = "$stage/bin;$env:PATH" + tsfile-cli.exe --version + + New-Item -ItemType Directory -Path consumer -Force | Out-Null + @' + cmake_minimum_required(VERSION 3.11) + project(TsFileInstalledConsumer LANGUAGES CXX) + + find_package(TsFile CONFIG REQUIRED) + add_executable(installed_consumer main.cpp) + target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) + '@ | Set-Content -Path consumer/CMakeLists.txt + @' + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + '@ | Set-Content -Path consumer/main.cpp + cmake -S consumer -B consumer/build -G "Visual Studio 17 2022" -A x64 ` + -DCMAKE_PREFIX_PATH="$stage" + cmake --build consumer/build --config Release --parallel + & consumer/build/Release/installed_consumer.exe + + - name: Build and inspect combined ZIP + env: + TSFILE_ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $cpackConfig = Get-Content build/windows/CPackConfig.cmake -Raw + if ($cpackConfig -notmatch 'set\(CPACK_ARCHIVE_COMPONENT_INSTALL "OFF"\)') { + throw 'CPACK_ARCHIVE_COMPONENT_INSTALL must remain disabled' + } + + New-Item -ItemType Directory -Path packages -Force | Out-Null + cpack -G ZIP --config build/windows/CPackConfig.cmake -B packages ` + -D "CPACK_PACKAGE_VERSION=$env:TSFILE_ARCHIVE_VERSION" + + $archives = @(Get-ChildItem -Path packages -Filter *.zip -File) + if ($archives.Count -ne 1) { + throw "Expected exactly one combined ZIP, found $($archives.Count)" + } + $expectedName = "tsfile-$env:TSFILE_ARCHIVE_VERSION-windows-x86_64.zip" + if ($archives[0].Name -cne $expectedName) { + throw "Unexpected ZIP name '$($archives[0].Name)'; expected '$expectedName'" + } + + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [System.IO.Compression.ZipFile]::OpenRead($archives[0].FullName) + try { + $entryPaths = @($archive.Entries | ForEach-Object { + $_.FullName.Replace('\', '/') + }) + $requiredSuffixes = @( + 'bin/tsfile-cli.exe' + 'bin/tsfile.dll' + 'lib/tsfile.lib' + 'include/tsfile/cwrapper/tsfile_cwrapper.h' + ) + foreach ($suffix in $requiredSuffixes) { + $match = $entryPaths | Where-Object { + $_.EndsWith($suffix, [System.StringComparison]::Ordinal) + } + if (-not $match) { + throw "ZIP is missing a path ending in '$suffix'" + } + } + } finally { + $archive.Dispose() + } + + - name: Upload Windows ZIP + uses: actions/upload-artifact@v7 + with: + name: native-windows-msvc-x86_64 + path: packages/*.zip + if-no-files-found: error + retention-days: 14 diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4b6f868b2..c7b486e00 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -545,7 +545,7 @@ unset(_TSFILE_PUBLIC_FEATURE) # target image provides every compatible dependency; AUTO remains the # reproducible fallback for distribution images with older packages. option(TSFILE_ENABLE_CPACK - "Enable CPack DEB/RPM package metadata generation" OFF) + "Enable CPack native package metadata generation" OFF) if (TSFILE_ENABLE_CPACK) set(CPACK_PACKAGE_NAME "tsfile") set(CPACK_PACKAGE_VENDOR "Apache Software Foundation") @@ -565,6 +565,13 @@ if (TSFILE_ENABLE_CPACK) set(CPACK_COMPONENTS_ALL runtime development tools) set(CPACK_COMPONENT_DEVELOPMENT_DEPENDS runtime) set(CPACK_COMPONENT_TOOLS_DEPENDS runtime) + # Portable archives contain the complete runtime, SDK, and tools install + # tree in one file. Native DEB and RPM generators remain componentized. + set(CPACK_ARCHIVE_COMPONENT_INSTALL OFF) + if (WIN32) + set(CPACK_PACKAGE_FILE_NAME + "tsfile-${TSFILE_ARCHIVE_VERSION}-windows-x86_64") + endif () set(CPACK_DEB_COMPONENT_INSTALL ON) set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) diff --git a/packaging/README.md b/packaging/README.md index 5cb218c78..8045a4f11 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -93,3 +93,19 @@ brew install ./packaging/homebrew/tsfile.rb The stable formula URL and checksum must be updated to the ASF source archive when the first release containing this packaging work is published. + +## Windows + +The manual workflow builds a 64-bit Release package with Visual Studio 2022 and +bundled dependencies. It stages and tests the CLI plus an external CMake SDK +consumer before producing +`tsfile--windows-x86_64.zip`. The archive is deliberately one +combined ZIP rather than one archive per CPack component, and contains the +runtime, development files, and tools under a relocatable prefix. In +particular, the MSVC outputs are installed as `bin/tsfile.dll`, +`lib/tsfile.lib`, and `bin/tsfile-cli.exe`. + +The workflow uploads that ZIP as the intermediate artifact +`native-windows-msvc-x86_64` for 14 days. Like the Linux jobs, it validates the +license files, CMake package config, staged executable, library, import library, +and public headers without publishing the artifact. From ba595ab023b1ba3ae2d872446b743977a1abb13e Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 19:07:24 +0800 Subject: [PATCH 09/27] fix(ci): package Windows Release artifacts --- .github/workflows/native-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 8b6bd3e4e..fdc560e3a 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -433,7 +433,7 @@ jobs: } New-Item -ItemType Directory -Path packages -Force | Out-Null - cpack -G ZIP --config build/windows/CPackConfig.cmake -B packages ` + cpack -G ZIP -C Release --config build/windows/CPackConfig.cmake -B packages ` -D "CPACK_PACKAGE_VERSION=$env:TSFILE_ARCHIVE_VERSION" $archives = @(Get-ChildItem -Path packages -Filter *.zip -File) From 19192b660d6471d1e56e4114de9a9b508f857ee2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 19:19:17 +0800 Subject: [PATCH 10/27] feat(ci): build Homebrew development bottles --- .github/workflows/native-packages.yml | 183 ++++++++++++++++++ packaging/README.md | 24 +++ packaging/homebrew/tsfile-dev.rb.in | 66 +++++++ packaging/scripts/render_homebrew_formula.py | 83 ++++++++ .../tests/test_render_homebrew_formula.py | 177 +++++++++++++++++ 5 files changed, 533 insertions(+) create mode 100644 packaging/homebrew/tsfile-dev.rb.in create mode 100644 packaging/scripts/render_homebrew_formula.py create mode 100644 packaging/tests/test_render_homebrew_formula.py diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index fdc560e3a..019918a70 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -476,3 +476,186 @@ jobs: path: packages/*.zip if-no-files-found: error retention-days: 14 + + build-homebrew: + name: Build Homebrew bottle on ${{ matrix.name }} + needs: prepare + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - name: macos-arm64 + os: macos-latest + - name: macos-x86_64 + os: macos-15-intel + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALL_CLEANUP: "1" + TSFILE_HOMEBREW_VERSION: ${{ needs.prepare.outputs.homebrew_version }} + SOURCE_REPOSITORY: ColinLeeo/tsfile + SOURCE_GIT_SHA: ${{ github.sha }} + BOTTLE_ROOT_URL: https://packages.apache.org/artifactory/tsfile/homebrew/dev/versions/${{ needs.prepare.outputs.homebrew_version }}/bottles + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Render immutable source Formula in a temporary local tap + shell: bash + run: | + mkdir -p homebrew/Formula homebrew/bottles + curl --fail --location --retry 3 \ + "https://github.com/$SOURCE_REPOSITORY/archive/$SOURCE_GIT_SHA.tar.gz" \ + --output "$RUNNER_TEMP/tsfile-source.tar.gz" + source_sha256="$(shasum -a 256 "$RUNNER_TEMP/tsfile-source.tar.gz" | awk '{print $1}')" + python3 packaging/scripts/render_homebrew_formula.py \ + --template packaging/homebrew/tsfile-dev.rb.in \ + --repository "$SOURCE_REPOSITORY" \ + --git-sha "$SOURCE_GIT_SHA" \ + --version "$TSFILE_HOMEBREW_VERSION" \ + --source-sha256 "$source_sha256" \ + --output homebrew/Formula/tsfile-dev.rb + ruby -c homebrew/Formula/tsfile-dev.rb + brew tap-new apache/tsfile-dev + tap_path="$(brew --repository apache/tsfile-dev)" + cp homebrew/Formula/tsfile-dev.rb "$tap_path/Formula/tsfile-dev.rb" + + - name: Build, test, and bottle the development Formula + shell: bash + run: | + brew install --build-bottle apache/tsfile-dev/tsfile-dev + brew test apache/tsfile-dev/tsfile-dev + cd homebrew/bottles + brew bottle --json --root-url "$BOTTLE_ROOT_URL" apache/tsfile-dev/tsfile-dev + shopt -s nullglob + bottles=(*.bottle*.tar.gz) + metadata=(*.bottle.json) + [[ ${#bottles[@]} -eq 1 && ${#metadata[@]} -eq 1 ]] + + - name: Upload platform Homebrew bottle and source Formula + uses: actions/upload-artifact@v7 + with: + name: native-homebrew-${{ matrix.name }} + path: homebrew/ + if-no-files-found: error + retention-days: 14 + + merge-homebrew: + name: Merge Homebrew development bottles + needs: [prepare, build-homebrew] + runs-on: macos-latest + timeout-minutes: 15 + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + TSFILE_HOMEBREW_VERSION: ${{ needs.prepare.outputs.homebrew_version }} + BOTTLE_ROOT_URL: https://packages.apache.org/artifactory/tsfile/homebrew/dev/versions/${{ needs.prepare.outputs.homebrew_version }}/bottles + steps: + - name: Download both platform Homebrew artifacts separately + uses: actions/download-artifact@v8 + with: + pattern: native-homebrew-macos-* + path: homebrew-intermediate + merge-multiple: false + + - name: Recreate the same pre-bottle Formula and collect metadata + shell: bash + run: | + arm=homebrew-intermediate/native-homebrew-macos-arm64 + intel=homebrew-intermediate/native-homebrew-macos-x86_64 + cmp "$arm/Formula/tsfile-dev.rb" "$intel/Formula/tsfile-dev.rb" + brew tap-new apache/tsfile-dev + tap_path="$(brew --repository apache/tsfile-dev)" + cp "$arm/Formula/tsfile-dev.rb" "$tap_path/Formula/tsfile-dev.rb" + mkdir -p homebrew/Formula homebrew/bottles + python3 - <<'PY' + import hashlib + import json + import os + import shutil + from pathlib import Path + + destination = Path("homebrew/bottles") + tags = set() + for platform in ("macos-arm64", "macos-x86_64"): + directory = Path("homebrew-intermediate") / f"native-homebrew-{platform}" / "bottles" + metadata = list(directory.glob("*.bottle.json")) + assert len(metadata) == 1, f"Expected one JSON for {platform}" + data = json.loads(metadata[0].read_text()) + assert list(data) == ["apache/tsfile-dev/tsfile-dev"] + entry = data["apache/tsfile-dev/tsfile-dev"] + assert entry["formula"]["pkg_version"] == os.environ["TSFILE_HOMEBREW_VERSION"] + assert entry["formula"]["path"] == "Library/Taps/apache/homebrew-tsfile-dev/Formula/tsfile-dev.rb" + assert entry["bottle"]["root_url"] == os.environ["BOTTLE_ROOT_URL"] + assert len(entry["bottle"]["tags"]) == 1 + tag, bottle = next(iter(entry["bottle"]["tags"].items())) + assert (platform == "macos-arm64") == tag.startswith("arm64_") + assert tag not in tags, f"Duplicate bottle tag: {tag}" + tags.add(tag) + local_name = bottle["local_filename"] + assert Path(local_name).name == local_name + source = directory / local_name + assert hashlib.sha256(source.read_bytes()).hexdigest() == bottle["sha256"] + for source in (source, metadata[0]): + target = destination / source.name + assert not target.exists(), f"Duplicate artifact: {target}" + shutil.copy2(source, target) + assert len(tags) == 2 + PY + + - name: Merge both bottle tags into the Formula + shell: bash + run: | + brew bottle --merge --write --no-commit homebrew/bottles/*.json + tap_path="$(brew --repository apache/tsfile-dev)" + cp "$tap_path/Formula/tsfile-dev.rb" homebrew/Formula/tsfile-dev.rb + ruby -c homebrew/Formula/tsfile-dev.rb + + - name: Verify merged Formula and normalize HTTP bottle filenames + shell: bash + run: | + python3 - <<'PY' + import hashlib + import json + import os + import re + from pathlib import Path + from urllib.parse import unquote + + formula = Path("homebrew/Formula/tsfile-dev.rb").read_text() + assert not re.search(r"@[A-Z0-9_]+@", formula), "Unresolved Formula token" + assert len(re.findall(r"^\s*bottle\s+do\b", formula, re.M)) == 1 + assert f'root_url "{os.environ["BOTTLE_ROOT_URL"]}"' in formula + directory = Path("homebrew/bottles") + metadata = list(directory.glob("*.bottle.json")) + assert len(metadata) == 2 + tags = set() + for path in metadata: + data = json.loads(path.read_text()) + entry = data["apache/tsfile-dev/tsfile-dev"] + for tag, bottle in entry["bottle"]["tags"].items(): + assert tag not in tags, f"Duplicate bottle tag: {tag}" + tags.add(tag) + assert re.search(rf'\b{re.escape(tag)}:\s*"{bottle["sha256"]}"', formula) + # Homebrew emits a double-dash local name but serves a single-dash URL. + filename = unquote(bottle["filename"]) + assert Path(filename).name == filename and filename.endswith(".tar.gz") + source = directory / bottle["local_filename"] + target = directory / filename + assert hashlib.sha256(source.read_bytes()).hexdigest() == bottle["sha256"] + if source != target: + assert not target.exists(), f"Duplicate HTTP bottle name: {filename}" + source.rename(target) + bottle["local_filename"] = filename + path.write_text(json.dumps(data, indent=2) + "\n") + assert len(tags) == 2 + assert len(list(directory.glob("*.tar.gz"))) == 2 + PY + + - name: Upload merged Homebrew Formula and bottles + uses: actions/upload-artifact@v7 + with: + name: native-homebrew + path: homebrew/ + if-no-files-found: error + retention-days: 14 diff --git a/packaging/README.md b/packaging/README.md index 8045a4f11..bf4afaee5 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -94,6 +94,30 @@ brew install ./packaging/homebrew/tsfile.rb The stable formula URL and checksum must be updated to the ASF source archive when the first release containing this packaging work is published. +The manual native-package workflow also builds the self-hosted development +Formula `tsfile-dev`, using `packaging/homebrew/tsfile-dev.rb.in`. This is not a +Homebrew/core submission. Each build pins the full workflow commit from +`ColinLeeo/tsfile`, hashes that source archive, and uses the generated immutable +Homebrew development version. The Formula retains the CMake install behavior +and tests the installed C++ consumer and CLI before bottling. + +The ARM64 job runs on `macos-latest` and the Intel job on `macos-15-intel`. +Their `native-homebrew-macos-arm64` and `native-homebrew-macos-x86_64` +intermediate artifacts remain separate until merge. The merge job checks that +both source Formula files match, merges both platform JSON files with Homebrew, +and checks both generated tags and checksums in the resulting Formula. + +The `native-homebrew` artifact contains `Formula/tsfile-dev.rb` and `bottles/` +with both bottle tarballs and both JSON metadata files. When downloaded into +`homebrew/`, this gives the final `homebrew/Formula/` and `homebrew/bottles/` +layout. The configured future bottle root is +`https://packages.apache.org/artifactory/tsfile/homebrew/dev/versions//bottles`. +Homebrew generates local tarballs with a double dash before the version but +requests a single dash in HTTP URLs. After merge, the workflow renames each +tarball to the JSON `filename` (URL-decoded) and updates its `local_filename` +to match, preserving its checksum. The workflow only uploads GitHub Actions +artifacts for 14 days; it does not publish to that root or update `latest`. + ## Windows The manual workflow builds a 64-bit Release package with Visual Studio 2022 and diff --git a/packaging/homebrew/tsfile-dev.rb.in b/packaging/homebrew/tsfile-dev.rb.in new file mode 100644 index 000000000..cc5b60bb2 --- /dev/null +++ b/packaging/homebrew/tsfile-dev.rb.in @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TsfileDev < Formula + desc "Columnar storage library for time series data (development snapshot)" + homepage "https://tsfile.apache.org/" + url "https://github.com/@SOURCE_REPOSITORY@/archive/@GIT_SHA@.tar.gz" + version "@HOMEBREW_VERSION@" + sha256 "@SOURCE_SHA256@" + license "Apache-2.0" + +@BOTTLE_BLOCK@ + + depends_on "cmake" => :build + depends_on "lz4" + depends_on "simde" + depends_on "snappy" + depends_on "utf8cpp" + depends_on "zstd" + + uses_from_macos "zlib" + + def install + args = %W[ + -DBUILD_TEST=OFF + -DBUILD_TOOLS=ON + -DENABLE_LZMA2=OFF + -DTSFILE_DEPENDENCY_SOURCE=AUTO + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + -DCMAKE_INSTALL_RPATH=#{rpath} + ] + + system "cmake", "-S", "cpp", "-B", "build", *args, *std_cmake_args + system "cmake", "--build", "build" + system "cmake", "--install", "build" + end + + test do + (testpath / "test.cpp").write <<~CPP + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + CPP + + system ENV.cxx, "-std=c++11", "test.cpp", "-I#{include}", "-L#{lib}", + "-Wl,-rpath,#{lib}", "-ltsfile", "-o", "test" + system "./test" + system bin / "tsfile-cli", "--version" + end +end diff --git a/packaging/scripts/render_homebrew_formula.py b/packaging/scripts/render_homebrew_formula.py new file mode 100644 index 000000000..a39ba31ee --- /dev/null +++ b/packaging/scripts/render_homebrew_formula.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Render the development Formula using an immutable source identity.""" + +import argparse +import re +from pathlib import Path + + +VALUE_PATTERNS = { + "SOURCE_REPOSITORY": r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "GIT_SHA": r"[0-9a-f]{40}", + "HOMEBREW_VERSION": r"[0-9][A-Za-z0-9.]*", + "SOURCE_SHA256": r"[0-9a-f]{64}", +} +TOKEN = re.compile(r"@([A-Z0-9_]+)@") + + +def render_formula( + template: str, values: dict[str, str], bottle_block: str = "" +) -> str: + """Replace declared tokens without interpreting Ruby braces or at-signs.""" + if values.keys() != VALUE_PATTERNS.keys(): + raise ValueError( + "Formula values must contain exactly the declared source fields" + ) + for name, pattern in VALUE_PATTERNS.items(): + if not re.fullmatch(pattern, values[name]): + raise ValueError(f"Invalid Formula value: {name}") + replacements = {**values, "BOTTLE_BLOCK": bottle_block} + unknown = set(TOKEN.findall(template)) - replacements.keys() + if unknown: + raise ValueError(f"Unknown Formula placeholders: {sorted(unknown)}") + rendered = TOKEN.sub(lambda match: replacements[match.group(1)], template) + if TOKEN.search(rendered): + raise ValueError("Unresolved Formula placeholder") + if len(re.findall(r"^\s*bottle\s+do\b", rendered, re.MULTILINE)) > 1: + raise ValueError("Duplicate bottle blocks") + return rendered + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--template", type=Path, required=True) + parser.add_argument("--repository", required=True) + parser.add_argument("--git-sha", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source-sha256", required=True) + parser.add_argument("--bottle-block", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + values = { + "SOURCE_REPOSITORY": args.repository, + "GIT_SHA": args.git_sha, + "HOMEBREW_VERSION": args.version, + "SOURCE_SHA256": args.source_sha256, + } + bottle_block = ( + args.bottle_block.read_text(encoding="utf-8") if args.bottle_block else "" + ) + args.output.write_text( + render_formula(args.template.read_text(encoding="utf-8"), values, bottle_block), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/test_render_homebrew_formula.py b/packaging/tests/test_render_homebrew_formula.py new file mode 100644 index 000000000..0c0047aed --- /dev/null +++ b/packaging/tests/test_render_homebrew_formula.py @@ -0,0 +1,177 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = ROOT / "packaging/scripts/render_homebrew_formula.py" +TEMPLATE_PATH = ROOT / "packaging/homebrew/tsfile-dev.rb.in" +VALUES = { + "SOURCE_REPOSITORY": "ColinLeeo/tsfile", + "GIT_SHA": "abcdef1234567890abcdef1234567890abcdef12", + "HOMEBREW_VERSION": "2.5.0.dev0.20260910.123.1.gabcdef1", + "SOURCE_SHA256": "0123456789abcdef" * 4, +} +TEMPLATE = """class TsfileDev < Formula + url "https://github.com/@SOURCE_REPOSITORY@/archive/@GIT_SHA@.tar.gz" + version "@HOMEBREW_VERSION@" + sha256 "@SOURCE_SHA256@" +@BOTTLE_BLOCK@ +end +""" +BOTTLE_BLOCK = """ bottle do + root_url "https://packages.apache.org/artifactory/tsfile/homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles" + sha256 cellar: :any, arm64_sequoia: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + sha256 cellar: :any, sequoia: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + end""" + + +class RenderHomebrewFormulaTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = None + if MODULE_PATH.is_file(): + spec = importlib.util.spec_from_file_location( + "render_homebrew_formula", MODULE_PATH + ) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def renderer(self): + self.assertIsNotNone(self.module, "Homebrew Formula renderer must exist") + return self.module.render_formula + + def test_renders_exact_immutable_source_and_both_bottle_tags(self): + rendered = self.renderer()(TEMPLATE, VALUES, BOTTLE_BLOCK) + self.assertEqual( + rendered, + """class TsfileDev < Formula + url "https://github.com/ColinLeeo/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz" + version "2.5.0.dev0.20260910.123.1.gabcdef1" + sha256 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +""" + + BOTTLE_BLOCK + + "\nend\n", + ) + + def test_pre_bottle_formula_has_no_bottle_block(self): + rendered = self.renderer()(TEMPLATE, VALUES) + self.assertNotIn("bottle do", rendered) + self.assertNotRegex(rendered, r"@[A-Z0-9_]+@") + + def test_rejects_missing_and_undeclared_placeholders(self): + render = self.renderer() + with self.assertRaises(ValueError): + render( + TEMPLATE, + {key: value for key, value in VALUES.items() if key != "GIT_SHA"}, + ) + with self.assertRaises(ValueError): + render(TEMPLATE + "@UNKNOWN@", {**VALUES, "UNKNOWN": "anything"}) + + def test_rejects_template_tokens_in_bottle_block(self): + with self.assertRaises(ValueError): + self.renderer()(TEMPLATE, VALUES, BOTTLE_BLOCK.replace("sequoia", "@TAG@")) + + def test_rejects_duplicate_bottle_blocks(self): + render = self.renderer() + for template, block in ( + (TEMPLATE, BOTTLE_BLOCK + "\n" + BOTTLE_BLOCK), + (TEMPLATE + "\nbottle do\nend\n", BOTTLE_BLOCK), + ( + TEMPLATE.replace("@BOTTLE_BLOCK@", "@BOTTLE_BLOCK@\n@BOTTLE_BLOCK@"), + BOTTLE_BLOCK, + ), + ): + with self.subTest(template=template, block=block): + with self.assertRaises(ValueError): + render(template, VALUES, block) + + def test_rejects_empty_or_non_immutable_source_identity(self): + render = self.renderer() + for key, value in ( + ("SOURCE_SHA256", ""), + ("SOURCE_SHA256", "bad"), + ("GIT_SHA", ""), + ("GIT_SHA", "abcdef1"), + ("GIT_SHA", "develop"), + ("SOURCE_REPOSITORY", 'ColinLeeo/tsfile"'), + ("HOMEBREW_VERSION", 'bad"\nversion "injected'), + ): + with self.subTest(key=key, value=value): + with self.assertRaises(ValueError): + render(TEMPLATE, {**VALUES, key: value}) + + def test_preserves_ruby_interpolation_and_other_at_signs(self): + extra = "\n# maintainer@example.org\n# #{rpath}\n" + self.assertTrue(self.renderer()(TEMPLATE + extra, VALUES).endswith(extra)) + + def test_cli_renders_real_template_with_optional_bottle_file(self): + self.renderer() + with tempfile.TemporaryDirectory() as temporary_directory: + directory = Path(temporary_directory) + output = directory / "tsfile-dev.rb" + block = directory / "bottle.rb" + block.write_text(BOTTLE_BLOCK, encoding="utf-8") + command = [ + sys.executable, + str(MODULE_PATH), + "--template", + str(TEMPLATE_PATH), + "--repository", + VALUES["SOURCE_REPOSITORY"], + "--git-sha", + VALUES["GIT_SHA"], + "--version", + VALUES["HOMEBREW_VERSION"], + "--source-sha256", + VALUES["SOURCE_SHA256"], + "--output", + str(output), + ] + for bottle_args, expected_block in ( + ([], ""), + (["--bottle-block", str(block)], BOTTLE_BLOCK), + ): + subprocess.run(command + bottle_args, check=True) + rendered = output.read_text(encoding="utf-8") + self.assertIn("class TsfileDev < Formula", rendered) + self.assertIn( + 'url "https://github.com/ColinLeeo/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz"', + rendered, + ) + self.assertIn('version "2.5.0.dev0.20260910.123.1.gabcdef1"', rendered) + self.assertIn( + 'sha256 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"', + rendered, + ) + self.assertIn("#{rpath}", rendered) + self.assertNotRegex(rendered, r"@[A-Z0-9_]+@") + if expected_block: + self.assertIn(BOTTLE_BLOCK, rendered) + else: + self.assertNotIn("bottle do", rendered) + + +if __name__ == "__main__": + unittest.main() From 340d0cd2a717d59492b0a4248fb39f8fe3af1007 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 20:51:37 +0800 Subject: [PATCH 11/27] feat(ci): assemble native package artifact bundle --- .github/workflows/native-packages.yml | 96 ++++++ packaging/README.md | 17 + packaging/scripts/assemble_native_packages.py | 244 ++++++++++++++ .../tests/test_assemble_native_packages.py | 298 ++++++++++++++++++ 4 files changed, 655 insertions(+) create mode 100644 packaging/scripts/assemble_native_packages.py create mode 100644 packaging/tests/test_assemble_native_packages.py diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 019918a70..dc01c872d 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -659,3 +659,99 @@ jobs: path: homebrew/ if-no-files-found: error retention-days: 14 + + assemble: + name: Assemble checksummed native package bundle + needs: [prepare, test-deb, test-rpm, merge-homebrew, build-windows] + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Download package versions + uses: actions/download-artifact@v8 + with: + name: native-package-versions + path: versions + + - name: Download DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-amd64 + path: input/deb + + - name: Download RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-x86_64 + path: input/rpm + + - name: Download Homebrew Formula and bottles + uses: actions/download-artifact@v8 + with: + name: native-homebrew + path: input/homebrew + + - name: Download Windows ZIP + uses: actions/download-artifact@v8 + with: + name: native-windows-msvc-x86_64 + path: input/windows + + - name: Assemble the publishable bundle + env: + LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }} + shell: bash + run: | + python3 packaging/scripts/assemble_native_packages.py \ + --input-dir input \ + --output-dir "tsfile-native-packages-$LOGICAL_VERSION" \ + --versions-json versions/versions.json \ + --source-repository "${{ github.repository }}" \ + --source-commit "${{ github.sha }}" + + - name: Verify assembled manifest checksums + env: + LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }} + shell: bash + run: | + python3 - <<'PY' + import hashlib + import json + import os + import re + from pathlib import Path + + output = Path(f"tsfile-native-packages-{os.environ['LOGICAL_VERSION']}") + manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) + checksums = (output / "SHA256SUMS").read_text(encoding="utf-8").splitlines() + expected_checksums = [] + paths = set() + for artifact in manifest["artifacts"]: + relative = Path(artifact["path"]) + assert relative.as_posix() == artifact["path"] + assert relative not in paths, f"Duplicate manifest path: {relative}" + paths.add(relative) + path = output / relative + assert path.is_file() and not path.is_symlink(), f"Missing artifact: {relative}" + digest = hashlib.sha256(path.read_bytes()).hexdigest() + assert re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"]) + assert digest == artifact["sha256"], f"Hash mismatch: {relative}" + assert path.stat().st_size == artifact["size"], f"Size mismatch: {relative}" + expected_checksums.append(f"{digest} {relative.as_posix()}") + assert checksums == sorted(expected_checksums) + actual_paths = { + path.relative_to(output) + for path in output.rglob("*") + if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"} + } + assert actual_paths == paths + PY + + - name: Upload checksummed native package bundle + uses: actions/upload-artifact@v7 + with: + name: tsfile-native-packages-${{ needs.prepare.outputs.archive_version }} + path: tsfile-native-packages-${{ needs.prepare.outputs.logical_version }}/ + if-no-files-found: error + retention-days: 14 diff --git a/packaging/README.md b/packaging/README.md index bf4afaee5..c693cd1f6 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -133,3 +133,20 @@ The workflow uploads that ZIP as the intermediate artifact `native-windows-msvc-x86_64` for 14 days. Like the Linux jobs, it validates the license files, CMake package config, staged executable, library, import library, and public headers without publishing the artifact. + +## Final native package bundle + +Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM +installation test, Homebrew bottle merge, and Windows SDK/CLI build all succeed, +the workflow assembles `tsfile-native-packages-`. The final +GitHub Actions artifact retains the DEBs, RPMs, merged Formula and bottles, and +Windows ZIP in their package-family layouts for 14 days. It also contains a +sorted `SHA256SUMS` and `manifest.json` with the source identity, generated +versions, byte sizes, SHA-256 values, and the JFrog repository, immutable target +path, and properties required for later manual publication. + +The final job has no publishing credentials and does not upload to JFrog. A +maintainer can later use the manifest to upload DEBs to `tsfile-debian` with the +recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, and Homebrew +and Windows files to their immutable `tsfile/homebrew/dev/versions/` +and `tsfile/windows/dev/versions/` paths. diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py new file mode 100644 index 000000000..d9c6a356c --- /dev/null +++ b/packaging/scripts/assemble_native_packages.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Assemble native package artifacts into a checksummed publishable bundle.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +from pathlib import Path +from typing import Sequence + +DEB_PROPERTIES = { + "deb.distribution": ["jammy", "noble"], + "deb.component": ["dev"], + "deb.architecture": ["amd64"], +} +REQUIRED_VERSIONS = {"archive_version", "homebrew_version"} +REQUIRED_SOURCE = {"commit", "repository"} + + +def _validate_metadata(values: dict[str, str], required: set[str], name: str) -> None: + if not isinstance(values, dict) or not required.issubset(values): + raise ValueError(f"{name} must contain {sorted(required)}") + if any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in values.items() + ): + raise ValueError(f"{name} must map strings to strings") + + +def _input_files(input_dir: Path) -> list[Path]: + if not input_dir.is_dir() or input_dir.is_symlink(): + raise ValueError(f"input directory is not a real directory: {input_dir}") + files: list[Path] = [] + for directory, directories, names in os.walk(input_dir, followlinks=False): + directory_path = Path(directory) + for child in directories: + path = directory_path / child + if path.is_symlink(): + raise ValueError(f"symbolic link input is not allowed: {path}") + for name in names: + path = directory_path / name + if path.is_symlink(): + raise ValueError(f"symbolic link input is not allowed: {path}") + if not path.is_file(): + raise ValueError(f"undeclared file type: {path}") + if path.stat().st_size == 0: + raise ValueError(f"empty file input is not allowed: {path}") + files.append(path) + return sorted(files, key=lambda path: path.relative_to(input_dir).as_posix()) + + +def _artifact_description( + input_dir: Path, source_path: Path, versions: dict[str, str] +) -> tuple[Path, dict[str, object]]: + relative_path = source_path.relative_to(input_dir) + parts = relative_path.parts + filename = source_path.name + if len(parts) >= 2 and parts[0] == "deb" and filename.endswith(".deb"): + target = Path("deb/ubuntu22.04-amd64") / filename + return target, { + "family": "deb", + "platform": "ubuntu22.04-amd64", + "targetRepository": "tsfile-debian", + "targetPath": f"pool/dev/ubuntu22.04-amd64/{filename}", + "properties": DEB_PROPERTIES, + } + if len(parts) >= 2 and parts[0] == "rpm" and filename.endswith(".rpm"): + target = Path("rpm/almalinux9-x86_64") / filename + return target, { + "family": "rpm", + "platform": "almalinux9-x86_64", + "targetRepository": "tsfile-rpm", + "targetPath": f"dev/el9/x86_64/{filename}", + "properties": {}, + } + if ( + len(parts) == 3 + and parts[:2] == ("homebrew", "Formula") + and filename == "tsfile-dev.rb" + ): + target = Path(*parts) + return target, { + "family": "homebrew", + "platform": "homebrew", + "targetRepository": "tsfile", + "targetPath": f"homebrew/dev/versions/{versions['homebrew_version']}/{target.as_posix()[len('homebrew/'):]}", + "properties": {}, + } + if ( + len(parts) == 3 + and parts[:2] == ("homebrew", "bottles") + and (filename.endswith(".tar.gz") or filename.endswith(".bottle.json")) + ): + target = Path(*parts) + return target, { + "family": "homebrew", + "platform": "homebrew", + "targetRepository": "tsfile", + "targetPath": f"homebrew/dev/versions/{versions['homebrew_version']}/{target.as_posix()[len('homebrew/'):]}", + "properties": {}, + } + if len(parts) >= 2 and parts[0] == "windows" and filename.endswith(".zip"): + target = Path("windows") / filename + return target, { + "family": "windows", + "platform": "windows-msvc-x86_64", + "targetRepository": "tsfile", + "targetPath": f"windows/dev/versions/{versions['archive_version']}/{filename}", + "properties": {}, + } + raise ValueError(f"undeclared file type: {relative_path.as_posix()}") + + +def _require_complete_families( + planned: list[tuple[Path, Path, dict[str, object]]], +) -> None: + families = [metadata["family"] for _, _, metadata in planned] + if not any(family == "deb" for family in families): + raise ValueError("missing DEB package input") + if not any(family == "rpm" for family in families): + raise ValueError("missing RPM package input") + if not any(family == "windows" for family in families): + raise ValueError("missing Windows ZIP input") + homebrew_paths = { + target.as_posix() + for _, target, metadata in planned + if metadata["family"] == "homebrew" + } + if "homebrew/Formula/tsfile-dev.rb" not in homebrew_paths: + raise ValueError("missing Homebrew Formula input") + if not any(path.endswith(".tar.gz") for path in homebrew_paths): + raise ValueError("missing Homebrew Bottle input") + if not any(path.endswith(".bottle.json") for path in homebrew_paths): + raise ValueError("missing Homebrew Bottle metadata input") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def assemble( + input_dir: Path, output_dir: Path, versions: dict[str, str], source: dict[str, str] +) -> dict[str, object]: + """Copy approved artifacts and return their deterministic publication manifest.""" + _validate_metadata(versions, REQUIRED_VERSIONS, "versions") + _validate_metadata(source, REQUIRED_SOURCE, "source") + input_dir = Path(input_dir) + output_dir = Path(output_dir) + files = _input_files(input_dir) + planned: list[tuple[Path, Path, dict[str, object]]] = [] + target_paths: set[Path] = set() + for source_path in files: + target, metadata = _artifact_description(input_dir, source_path, versions) + if target in target_paths: + raise ValueError(f"duplicate target path: {target.as_posix()}") + target_paths.add(target) + planned.append((source_path, target, metadata)) + _require_complete_families(planned) + if output_dir.is_symlink(): + raise ValueError(f"output directory must not be a symbolic link: {output_dir}") + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError(f"output directory must be empty: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + + artifacts: list[dict[str, object]] = [] + for source_path, target, metadata in sorted( + planned, key=lambda item: item[1].as_posix() + ): + destination = output_dir / target + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source_path, destination) + artifact = { + **metadata, + "filename": destination.name, + "path": target.as_posix(), + "size": destination.stat().st_size, + "sha256": _sha256(destination), + } + artifacts.append(artifact) + + checksums = "".join( + f"{artifact['sha256']} {artifact['path']}\n" for artifact in artifacts + ) + (output_dir / "SHA256SUMS").write_text(checksums, encoding="utf-8") + manifest: dict[str, object] = { + "artifacts": artifacts, + "source": source, + "versions": versions, + } + (output_dir / "manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def _argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--versions-json", type=Path, required=True) + parser.add_argument("--source-repository", required=True) + parser.add_argument("--source-commit", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Assemble a bundle from an Actions artifact download directory.""" + arguments = _argument_parser().parse_args(argv) + versions = json.loads(arguments.versions_json.read_text(encoding="utf-8")) + assemble( + arguments.input_dir, + arguments.output_dir, + versions, + {"repository": arguments.source_repository, "commit": arguments.source_commit}, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py new file mode 100644 index 000000000..ab2342378 --- /dev/null +++ b/packaging/tests/test_assemble_native_packages.py @@ -0,0 +1,298 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = REPOSITORY_ROOT / "packaging/scripts/assemble_native_packages.py" +VERSIONS = { + "archive_version": "2.5.0-dev0.20260910.123.1.gabcdef1", + "homebrew_version": "2.5.0.dev0.20260910.123.1.gabcdef1", + "logical_version": "2.5.0.dev0+20260910.123.1.gabcdef1", +} +SOURCE = { + "commit": "abcdef1234567890abcdef1234567890abcdef12", + "repository": "ColinLeeo/tsfile", +} + + +def load_assembler_module(): + if not MODULE_PATH.is_file(): + return None + spec = importlib.util.spec_from_file_location( + "assemble_native_packages", MODULE_PATH + ) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load native package assembler from {MODULE_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class AssembleNativePackagesTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = load_assembler_module() + + def require_module(self): + self.assertIsNotNone( + self.module, "native package assembler must exist for final bundle assembly" + ) + return self.module + + def write_fixture(self, directory): + files = { + "deb/tsfile_2.5.0-dev_amd64.deb": b"deb artifact\n", + "rpm/tsfile-2.5.0-dev.x86_64.rpm": b"rpm artifact\n", + "homebrew/Formula/tsfile-dev.rb": b"formula artifact\n", + "homebrew/bottles/tsfile-dev.bottle.tar.gz": b"bottle artifact\n", + "homebrew/bottles/tsfile-dev.bottle.json": b"bottle metadata\n", + "windows/tsfile-2.5.0-dev-windows-x86_64.zip": b"windows artifact\n", + } + for relative_path, contents in files.items(): + path = directory / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + return files + + def test_assembles_publishable_bundle_with_literal_metadata(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + output_directory = Path(temporary_directory) / "bundle" + self.write_fixture(input_directory) + + manifest = module.assemble( + input_directory, output_directory, VERSIONS, SOURCE + ) + + expected_paths = [ + "deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", + "homebrew/Formula/tsfile-dev.rb", + "homebrew/bottles/tsfile-dev.bottle.json", + "homebrew/bottles/tsfile-dev.bottle.tar.gz", + "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "windows/tsfile-2.5.0-dev-windows-x86_64.zip", + ] + self.assertEqual( + sorted( + path.relative_to(output_directory).as_posix() + for path in output_directory.rglob("*") + if path.is_file() + and path.name not in {"SHA256SUMS", "manifest.json"} + ), + expected_paths, + ) + self.assertEqual( + (output_directory / "SHA256SUMS").read_text(encoding="utf-8"), + "ef5ca1431457b6dec117aff4aafcfd14edec3fe628d3fefe334867e502b8c137 deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb\n" + "b045d33311f553503342e1b061df60161e414503d68dc14af90537f254dfc224 homebrew/Formula/tsfile-dev.rb\n" + "ae98b99fb84c92fb10b00d75e0c51035006067659043b9dc0cb08d9902633967 homebrew/bottles/tsfile-dev.bottle.json\n" + "95f3ddb71b1a0c5c95c5aa0352321075cc465d6a583dff8bf601e888abec4b82 homebrew/bottles/tsfile-dev.bottle.tar.gz\n" + "f557fbfdaf15a34e59adaf4fa487062419dd8fe39eb057f6b581b4469cd3a25f rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm\n" + "f59fc30fd2aaa4a4594eb006fce33041e2702b621abf936af5619b3eabf23c8f windows/tsfile-2.5.0-dev-windows-x86_64.zip\n", + ) + self.assertEqual( + json.loads((output_directory / "manifest.json").read_text()), manifest + ) + self.assertEqual(manifest["source"], SOURCE) + self.assertEqual(manifest["versions"], VERSIONS) + self.assertEqual( + manifest["artifacts"], + [ + { + "family": "deb", + "filename": "tsfile_2.5.0-dev_amd64.deb", + "path": "deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", + "platform": "ubuntu22.04-amd64", + "properties": { + "deb.architecture": ["amd64"], + "deb.component": ["dev"], + "deb.distribution": ["jammy", "noble"], + }, + "sha256": "ef5ca1431457b6dec117aff4aafcfd14edec3fe628d3fefe334867e502b8c137", + "size": 13, + "targetPath": "pool/dev/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", + "targetRepository": "tsfile-debian", + }, + { + "family": "homebrew", + "filename": "tsfile-dev.rb", + "path": "homebrew/Formula/tsfile-dev.rb", + "platform": "homebrew", + "properties": {}, + "sha256": "b045d33311f553503342e1b061df60161e414503d68dc14af90537f254dfc224", + "size": 17, + "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/Formula/tsfile-dev.rb", + "targetRepository": "tsfile", + }, + { + "family": "homebrew", + "filename": "tsfile-dev.bottle.json", + "path": "homebrew/bottles/tsfile-dev.bottle.json", + "platform": "homebrew", + "properties": {}, + "sha256": "ae98b99fb84c92fb10b00d75e0c51035006067659043b9dc0cb08d9902633967", + "size": 16, + "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles/tsfile-dev.bottle.json", + "targetRepository": "tsfile", + }, + { + "family": "homebrew", + "filename": "tsfile-dev.bottle.tar.gz", + "path": "homebrew/bottles/tsfile-dev.bottle.tar.gz", + "platform": "homebrew", + "properties": {}, + "sha256": "95f3ddb71b1a0c5c95c5aa0352321075cc465d6a583dff8bf601e888abec4b82", + "size": 16, + "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles/tsfile-dev.bottle.tar.gz", + "targetRepository": "tsfile", + }, + { + "family": "rpm", + "filename": "tsfile-2.5.0-dev.x86_64.rpm", + "path": "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "platform": "almalinux9-x86_64", + "properties": {}, + "sha256": "f557fbfdaf15a34e59adaf4fa487062419dd8fe39eb057f6b581b4469cd3a25f", + "size": 13, + "targetPath": "dev/el9/x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "targetRepository": "tsfile-rpm", + }, + { + "family": "windows", + "filename": "tsfile-2.5.0-dev-windows-x86_64.zip", + "path": "windows/tsfile-2.5.0-dev-windows-x86_64.zip", + "platform": "windows-msvc-x86_64", + "properties": {}, + "sha256": "f59fc30fd2aaa4a4594eb006fce33041e2702b621abf936af5619b3eabf23c8f", + "size": 17, + "targetPath": "windows/dev/versions/2.5.0-dev0.20260910.123.1.gabcdef1/tsfile-2.5.0-dev-windows-x86_64.zip", + "targetRepository": "tsfile", + }, + ], + ) + + def test_rejects_unknown_files(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + (input_directory / "deb/README.txt").write_text( + "not a package\n", encoding="utf-8" + ) + + with self.assertRaisesRegex(ValueError, "undeclared file type"): + module.assemble( + input_directory, + Path(temporary_directory) / "bundle", + VERSIONS, + SOURCE, + ) + + def test_rejects_empty_and_symbolic_link_inputs(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + (input_directory / "rpm/tsfile-2.5.0-dev.x86_64.rpm").write_bytes(b"") + with self.assertRaisesRegex(ValueError, "empty file"): + module.assemble( + input_directory, + Path(temporary_directory) / "empty", + VERSIONS, + SOURCE, + ) + + self.write_fixture(input_directory) + os.symlink( + input_directory / "deb/tsfile_2.5.0-dev_amd64.deb", + input_directory / "deb/linked.deb", + ) + with self.assertRaisesRegex(ValueError, "symbolic link"): + module.assemble( + input_directory, + Path(temporary_directory) / "linked", + VERSIONS, + SOURCE, + ) + + def test_rejects_duplicate_destination_names(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + duplicate = input_directory / "deb/duplicate/tsfile_2.5.0-dev_amd64.deb" + duplicate.parent.mkdir() + duplicate.write_bytes(b"another deb artifact\n") + + with self.assertRaisesRegex(ValueError, "duplicate target path"): + module.assemble( + input_directory, + Path(temporary_directory) / "bundle", + VERSIONS, + SOURCE, + ) + + def test_cli_assembles_from_versions_json(self): + self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_path = Path(temporary_directory) + input_directory = temporary_path / "input" + self.write_fixture(input_directory) + versions_path = temporary_path / "versions.json" + versions_path.write_text(json.dumps(VERSIONS), encoding="utf-8") + output_directory = temporary_path / "bundle" + + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--input-dir", + str(input_directory), + "--output-dir", + str(output_directory), + "--versions-json", + str(versions_path), + "--source-repository", + SOURCE["repository"], + "--source-commit", + SOURCE["commit"], + ], + check=True, + ) + manifest_path = output_directory / "manifest.json" + self.assertTrue(manifest_path.is_file(), "CLI must write manifest.json") + self.assertEqual( + json.loads(manifest_path.read_text(encoding="utf-8"))["source"], SOURCE + ) + + +if __name__ == "__main__": + unittest.main() From 6ffc6c0da3be3acbf95e0934bd7ae3063d1db8da Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 20:58:36 +0800 Subject: [PATCH 12/27] fix(ci): verify native package bundle integrity --- .github/workflows/native-packages.yml | 34 +---- packaging/scripts/assemble_native_packages.py | 134 +++++++++++++++++- .../tests/test_assemble_native_packages.py | 57 ++++++++ 3 files changed, 186 insertions(+), 39 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index dc01c872d..3c326d2e2 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -715,38 +715,8 @@ jobs: LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }} shell: bash run: | - python3 - <<'PY' - import hashlib - import json - import os - import re - from pathlib import Path - - output = Path(f"tsfile-native-packages-{os.environ['LOGICAL_VERSION']}") - manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) - checksums = (output / "SHA256SUMS").read_text(encoding="utf-8").splitlines() - expected_checksums = [] - paths = set() - for artifact in manifest["artifacts"]: - relative = Path(artifact["path"]) - assert relative.as_posix() == artifact["path"] - assert relative not in paths, f"Duplicate manifest path: {relative}" - paths.add(relative) - path = output / relative - assert path.is_file() and not path.is_symlink(), f"Missing artifact: {relative}" - digest = hashlib.sha256(path.read_bytes()).hexdigest() - assert re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"]) - assert digest == artifact["sha256"], f"Hash mismatch: {relative}" - assert path.stat().st_size == artifact["size"], f"Size mismatch: {relative}" - expected_checksums.append(f"{digest} {relative.as_posix()}") - assert checksums == sorted(expected_checksums) - actual_paths = { - path.relative_to(output) - for path in output.rglob("*") - if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"} - } - assert actual_paths == paths - PY + python3 packaging/scripts/assemble_native_packages.py \ + --verify-bundle "tsfile-native-packages-$LOGICAL_VERSION" - name: Upload checksummed native package bundle uses: actions/upload-artifact@v7 diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py index d9c6a356c..51cec3769 100644 --- a/packaging/scripts/assemble_native_packages.py +++ b/packaging/scripts/assemble_native_packages.py @@ -24,6 +24,7 @@ import hashlib import json import os +import re import shutil from pathlib import Path from typing import Sequence @@ -47,6 +48,27 @@ def _validate_metadata(values: dict[str, str], required: set[str], name: str) -> raise ValueError(f"{name} must map strings to strings") +def _validate_versions(versions: dict[str, str]) -> None: + _validate_metadata(versions, REQUIRED_VERSIONS, "versions") + for name in REQUIRED_VERSIONS: + value = versions[name] + if ( + not value.strip() + or value in {".", ".."} + or "/" in value + or "\\" in value + or "\x00" in value + ): + raise ValueError(f"version {name} must be a safe path component") + + +def _validate_source(source: dict[str, str]) -> None: + _validate_metadata(source, REQUIRED_SOURCE, "source") + for name in REQUIRED_SOURCE: + if not source[name].strip(): + raise ValueError(f"source {name} must be nonempty") + + def _input_files(input_dir: Path) -> list[Path]: if not input_dir.is_dir() or input_dir.is_symlink(): raise ValueError(f"input directory is not a real directory: {input_dir}") @@ -162,12 +184,80 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _artifact_paths(output_dir: Path) -> set[Path]: + paths: set[Path] = set() + for path in output_dir.rglob("*"): + if path.is_symlink(): + raise ValueError(f"symbolic link output is not allowed: {path}") + if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"}: + paths.add(path.relative_to(output_dir)) + return paths + + +def verify_bundle(output_dir: Path) -> None: + """Verify manifest hashes, sizes, and path-sorted checksums in a bundle.""" + output_dir = Path(output_dir) + if not output_dir.is_dir() or output_dir.is_symlink(): + raise ValueError(f"bundle directory is not a real directory: {output_dir}") + manifest_path = output_dir / "manifest.json" + checksums_path = output_dir / "SHA256SUMS" + if ( + not manifest_path.is_file() + or manifest_path.is_symlink() + or not checksums_path.is_file() + or checksums_path.is_symlink() + ): + raise ValueError("bundle must contain real manifest.json and SHA256SUMS files") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + artifacts = manifest.get("artifacts") if isinstance(manifest, dict) else None + if not isinstance(artifacts, list): + raise ValueError("manifest artifacts must be a list") + + paths: set[Path] = set() + for artifact in artifacts: + if not isinstance(artifact, dict): + raise ValueError("manifest artifacts must be objects") + relative_text = artifact.get("path") + if not isinstance(relative_text, str): + raise ValueError("manifest artifact path must be a string") + relative = Path(relative_text) + if ( + relative.is_absolute() + or relative.as_posix() != relative_text + or ".." in relative.parts + ): + raise ValueError(f"unsafe manifest path: {relative_text}") + if relative in paths: + raise ValueError(f"duplicate manifest path: {relative_text}") + paths.add(relative) + path = output_dir / relative + if not path.is_file() or path.is_symlink(): + raise ValueError(f"missing artifact: {relative_text}") + sha256 = artifact.get("sha256") + if not isinstance(sha256, str) or re.fullmatch(r"[0-9a-f]{64}", sha256) is None: + raise ValueError(f"invalid artifact SHA-256: {relative_text}") + if _sha256(path) != sha256: + raise ValueError(f"hash mismatch: {relative_text}") + if path.stat().st_size != artifact.get("size"): + raise ValueError(f"size mismatch: {relative_text}") + + actual_checksums = checksums_path.read_text(encoding="utf-8").splitlines() + expected_checksums = [ + f"{artifact['sha256']} {artifact['path']}" + for artifact in sorted(artifacts, key=lambda artifact: artifact["path"]) + ] + if actual_checksums != expected_checksums: + raise ValueError("SHA256SUMS does not match path-sorted manifest artifacts") + if _artifact_paths(output_dir) != paths: + raise ValueError("bundle files do not match manifest artifacts") + + def assemble( input_dir: Path, output_dir: Path, versions: dict[str, str], source: dict[str, str] ) -> dict[str, object]: """Copy approved artifacts and return their deterministic publication manifest.""" - _validate_metadata(versions, REQUIRED_VERSIONS, "versions") - _validate_metadata(source, REQUIRED_SOURCE, "source") + _validate_versions(versions) + _validate_source(source) input_dir = Path(input_dir) output_dir = Path(output_dir) files = _input_files(input_dir) @@ -219,17 +309,47 @@ def assemble( def _argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--input-dir", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--versions-json", type=Path, required=True) - parser.add_argument("--source-repository", required=True) - parser.add_argument("--source-commit", required=True) + parser.add_argument("--input-dir", type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--versions-json", type=Path) + parser.add_argument("--source-repository") + parser.add_argument("--source-commit") + parser.add_argument("--verify-bundle", type=Path) return parser def main(argv: Sequence[str] | None = None) -> int: """Assemble a bundle from an Actions artifact download directory.""" arguments = _argument_parser().parse_args(argv) + if arguments.verify_bundle is not None: + if any( + value is not None + for value in ( + arguments.input_dir, + arguments.output_dir, + arguments.versions_json, + arguments.source_repository, + arguments.source_commit, + ) + ): + raise ValueError( + "--verify-bundle cannot be combined with assembly arguments" + ) + verify_bundle(arguments.verify_bundle) + return 0 + missing = [ + name + for name, value in ( + ("--input-dir", arguments.input_dir), + ("--output-dir", arguments.output_dir), + ("--versions-json", arguments.versions_json), + ("--source-repository", arguments.source_repository), + ("--source-commit", arguments.source_commit), + ) + if value is None + ] + if missing: + raise ValueError(f"missing assembly arguments: {', '.join(missing)}") versions = json.loads(arguments.versions_json.read_text(encoding="utf-8")) assemble( arguments.input_dir, diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py index ab2342378..54da32c66 100644 --- a/packaging/tests/test_assemble_native_packages.py +++ b/packaging/tests/test_assemble_native_packages.py @@ -195,6 +195,63 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): ], ) + def test_verifies_assembler_checksum_lines_sorted_by_path(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + output_directory = Path(temporary_directory) / "bundle" + self.write_fixture(input_directory) + module.assemble(input_directory, output_directory, VERSIONS, SOURCE) + + checksum_lines = ( + (output_directory / "SHA256SUMS") + .read_text(encoding="utf-8") + .splitlines() + ) + self.assertNotEqual( + checksum_lines, + sorted(checksum_lines), + "fixture must distinguish path ordering from hash ordering", + ) + self.assertIsNone(module.verify_bundle(output_directory)) + + def test_rejects_unsafe_versions_and_empty_source_identity(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + for index, (field, value) in enumerate( + ( + ("archive_version", "../../latest"), + ("homebrew_version", "../latest"), + ("archive_version", ""), + ("homebrew_version", "version\\latest"), + ) + ): + with self.subTest(field=field, value=value): + versions = {**VERSIONS, field: value} + with self.assertRaisesRegex(ValueError, "safe path component"): + module.assemble( + input_directory, + Path(temporary_directory) / f"invalid-{index}", + versions, + SOURCE, + ) + for index, (field, value) in enumerate( + (("commit", ""), ("repository", " ")) + ): + with self.subTest(field=field, value=value): + source = {**SOURCE, field: value} + with self.assertRaisesRegex(ValueError, "nonempty"): + module.assemble( + input_directory, + Path(temporary_directory) / f"empty-{index}", + VERSIONS, + source, + ) + def test_rejects_unknown_files(self): module = self.require_module() From 205a813d53b3b79df8de1eb018b1e19d26cf6472 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 21:12:20 +0800 Subject: [PATCH 13/27] fix(packaging): omit empty Homebrew bottle spacing --- packaging/scripts/render_homebrew_formula.py | 3 +++ packaging/tests/test_render_homebrew_formula.py | 1 + 2 files changed, 4 insertions(+) diff --git a/packaging/scripts/render_homebrew_formula.py b/packaging/scripts/render_homebrew_formula.py index a39ba31ee..60217917e 100644 --- a/packaging/scripts/render_homebrew_formula.py +++ b/packaging/scripts/render_homebrew_formula.py @@ -46,6 +46,9 @@ def render_formula( unknown = set(TOKEN.findall(template)) - replacements.keys() if unknown: raise ValueError(f"Unknown Formula placeholders: {sorted(unknown)}") + if not bottle_block: + # Omit an empty standalone block and its following separator line. + template = re.sub(r"(?m)^[ \t]*@BOTTLE_BLOCK@\n(?:[ \t]*\n)?", "", template) rendered = TOKEN.sub(lambda match: replacements[match.group(1)], template) if TOKEN.search(rendered): raise ValueError("Unresolved Formula placeholder") diff --git a/packaging/tests/test_render_homebrew_formula.py b/packaging/tests/test_render_homebrew_formula.py index 0c0047aed..a702c40d9 100644 --- a/packaging/tests/test_render_homebrew_formula.py +++ b/packaging/tests/test_render_homebrew_formula.py @@ -167,6 +167,7 @@ def test_cli_renders_real_template_with_optional_bottle_file(self): ) self.assertIn("#{rpath}", rendered) self.assertNotRegex(rendered, r"@[A-Z0-9_]+@") + self.assertNotIn("\n\n\n", rendered) if expected_block: self.assertIn(BOTTLE_BLOCK, rendered) else: From b8874a02869573c30ec669ad5d7c6f9b245e2e8b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 21:12:34 +0800 Subject: [PATCH 14/27] docs: document native package artifact workflow --- .github/workflows/cpp-packaging.yml | 128 ---------------------------- packaging/README.md | 30 +++++-- 2 files changed, 24 insertions(+), 134 deletions(-) delete mode 100644 .github/workflows/cpp-packaging.yml diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml deleted file mode 100644 index 54fa92211..000000000 --- a/.github/workflows/cpp-packaging.yml +++ /dev/null @@ -1,128 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Cpp-Packaging - -on: - push: - branches: - - develop - - rc/** - paths: - - '.github/workflows/cpp-packaging.yml' - - 'cpp/**' - - 'packaging/**' - - 'LICENSE' - - 'NOTICE' - pull_request: - branches: - - develop - - rc/** - paths: - - '.github/workflows/cpp-packaging.yml' - - 'cpp/**' - - 'packaging/**' - - 'LICENSE' - - 'NOTICE' - workflow_dispatch: - -permissions: - contents: read - -jobs: - deb: - name: Debian package - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Install packaging tools - run: | - sudo apt-get update - sudo apt-get install -y build-essential cmake ninja-build pkg-config dpkg-dev uuid-dev - - - name: Configure and build - run: | - cmake -S cpp -B build/deb -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST=OFF \ - -DBUILD_TOOLS=ON \ - -DTSFILE_ENABLE_CPACK=ON \ - -DTSFILE_DEPENDENCY_SOURCE=AUTO \ - -DTSFILE_ENABLE_NATIVE_ARCH=OFF - cmake --build build/deb --parallel - - - name: Build and inspect DEB packages - run: | - cpack --config build/deb/CPackConfig.cmake -G DEB - ls -lh ./*.deb - for package in ./*.deb; do - dpkg-deb --info "$package" - dpkg-deb --contents "$package" | grep -E '(/libtsfile|/tsfile-cli|TsFileConfig|libtsfile.pc)' || true - done - - - name: Upload DEB packages - uses: actions/upload-artifact@v4 - with: - name: tsfile-deb - path: '*.deb' - if-no-files-found: error - - rpm: - name: Fedora package - runs-on: ubuntu-24.04 - container: fedora:latest - timeout-minutes: 30 - steps: - - name: Install build and packaging tools - run: | - dnf install -y \ - cmake gcc-c++ make ninja-build pkgconf-pkg-config rpm-build \ - git curl tar xz unzip gzip libuuid-devel - - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Configure and build - run: | - cmake -S cpp -B build/rpm -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST=OFF \ - -DBUILD_TOOLS=ON \ - -DTSFILE_ENABLE_CPACK=ON \ - -DTSFILE_DEPENDENCY_SOURCE=AUTO \ - -DTSFILE_ENABLE_NATIVE_ARCH=OFF - cmake --build build/rpm --parallel - - - name: Build and inspect RPM packages - run: | - cpack --config build/rpm/CPackConfig.cmake -G RPM - ls -lh ./*.rpm - for package in ./*.rpm; do - rpm -qip "$package" - rpm -qlp "$package" | grep -E '(/libtsfile|/tsfile-cli|TsFileConfig|libtsfile.pc)' || true - done - - - name: Upload RPM packages - uses: actions/upload-artifact@v4 - with: - name: tsfile-rpm - path: '*.rpm' - if-no-files-found: error diff --git a/packaging/README.md b/packaging/README.md index c693cd1f6..910026236 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -29,9 +29,29 @@ The header closure mirrors the include relationships used by the current C++ implementation. It is intentionally broader than the long-term stable public API; a separate API cleanup will narrow it in a future version. +## Manual artifact workflow + +`Build native package artifacts` (`.github/workflows/native-packages.yml`) is +the authoritative native packaging workflow. In the fork's GitHub Actions tab, +select this workflow, choose **Run workflow**, select the branch containing the +commit to build, and dispatch it manually. It runs only on `workflow_dispatch`, +with read-only repository permissions; pushes and pull requests do not trigger +native packaging. + +A successful run produces Ubuntu DEBs, AlmaLinux RPMs, an ARM64 and Intel +Homebrew development bottle with a merged Formula, and a Windows x86_64 SDK/CLI +ZIP. It combines these into `tsfile-native-packages-` with +`manifest.json` and `SHA256SUMS`. Download this final artifact from the workflow +run; both intermediate and final artifacts are retained for 14 days. + +Publishing is a separate, manual step. This workflow only builds, tests, and +uploads GitHub Actions artifacts. It has no publishing credentials and does +not upload to JFrog, sign packages, update `latest`, create tags or releases, or +implement RC/final release behavior. + ## Portable archive -Build a relocatable source archive on any host with CMake and CPack: +Build a relocatable binary archive on any host with CMake and CPack: ```bash cmake -S cpp -B cpp/build/package \ @@ -41,7 +61,6 @@ cmake -S cpp -B cpp/build/package \ -DTSFILE_ENABLE_CPACK=ON \ -DTSFILE_DEPENDENCY_SOURCE=AUTO cmake --build cpp/build/package --parallel -cmake --install cpp/build/package cpack --config cpp/build/package/CPackConfig.cmake -G TGZ ``` @@ -63,15 +82,14 @@ cmake -S cpp -B cpp/build/package \ -DTSFILE_ENABLE_CPACK=ON \ -DTSFILE_DEPENDENCY_SOURCE=AUTO cmake --build cpp/build/package --parallel -cmake --install cpp/build/package cpack --config cpp/build/package/CPackConfig.cmake -G DEB # or: cpack --config cpp/build/package/CPackConfig.cmake -G RPM ``` `SYSTEM` can be used instead of `AUTO` when the build image provides every -compatible dependency, including ANTLR4 4.9.x. `AUTO` is the reproducible -release default and uses the verified source fallback for unavailable or -incompatible distro versions. +compatible dependency, including ANTLR4 >=4.9.3 and <4.13.0. `AUTO` is the +reproducible release default and uses the verified source fallback for +unavailable or incompatible distro versions. The manual `Build native package artifacts` workflow is the authoritative native package build. Its Linux jobs create `tsfile`, `tsfile-dev`, and From 868114503797387c3a468599897043d0050247cb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 22:08:17 +0800 Subject: [PATCH 15/27] fix(packaging): complete native runtime and install contracts --- .github/workflows/native-packages.yml | 85 +++----- cpp/CMakeLists.txt | 27 +++ cpp/cmake/TsFileConfig.cmake.in | 14 ++ cpp/cmake/TsFileStaticDependencies.cmake | 104 +++++++++ cpp/cmake/TsFileStaticDependencies.cmake.in | 20 ++ cpp/cmake/libtsfile.pc.in | 2 +- cpp/cmake/tests/CheckStaticMSVCRuntime.cmake | 38 ++++ .../projects/InstalledConsumer/CMakeLists.txt | 3 + .../tests/projects/InstalledConsumer/main.cc | 5 +- .../tests/projects/PkgConfigConsumer/main.c | 5 +- .../tests/projects/PkgConfigConsumer/main.cc | 6 +- cpp/src/CMakeLists.txt | 13 +- packaging/README.md | 41 ++++ packaging/homebrew/tsfile-dev.rb.in | 3 +- packaging/homebrew/tsfile.rb | 3 +- packaging/scripts/native_package_versions.py | 11 +- packaging/scripts/verify_windows_runtime.py | 101 +++++++++ packaging/tests/check_native_workflow.rb | 53 +++++ packaging/tests/test_native_install.py | 206 ++++++++++++++++++ .../tests/test_native_package_versions.py | 41 ++++ packaging/tests/test_windows_runtime.py | 73 +++++++ 21 files changed, 788 insertions(+), 66 deletions(-) create mode 100644 cpp/cmake/TsFileStaticDependencies.cmake create mode 100644 cpp/cmake/TsFileStaticDependencies.cmake.in create mode 100644 cpp/cmake/tests/CheckStaticMSVCRuntime.cmake create mode 100644 packaging/scripts/verify_windows_runtime.py create mode 100644 packaging/tests/check_native_workflow.rb create mode 100644 packaging/tests/test_native_install.py create mode 100644 packaging/tests/test_windows_runtime.py diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 3c326d2e2..f6f349315 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -42,6 +42,11 @@ jobs: with: fetch-depth: 0 + - name: Check packaging helpers and workflow contracts + run: | + python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v + ruby packaging/tests/check_native_workflow.rb + - name: Generate package versions id: versions run: | @@ -152,6 +157,9 @@ jobs: - ubuntu:24.04 container: ${{ matrix.container }} steps: + - name: Checkout installed-consumer fixture + uses: actions/checkout@v7 + - name: Download DEB packages uses: actions/download-artifact@v8 with: @@ -181,25 +189,9 @@ jobs: - name: Build and run an installed-package consumer shell: bash run: | - mkdir -p consumer - cat > consumer/CMakeLists.txt <<'CMAKE' - cmake_minimum_required(VERSION 3.11) - project(TsFileInstalledConsumer LANGUAGES CXX) - - find_package(TsFile CONFIG REQUIRED) - add_executable(installed_consumer main.cpp) - target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) - CMAKE - cat > consumer/main.cpp <<'CPP' - #include - - int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; - } - CPP - cmake -S consumer -B consumer/build + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build cmake --build consumer/build --parallel - ./consumer/build/installed_consumer + ./consumer/build/tsfile_installed_consumer build-rpm: name: Build AlmaLinux 9 RPM packages @@ -217,7 +209,7 @@ jobs: pkgconf-pkg-config \ rpm-build \ git \ - curl \ + curl-minimal \ tar \ gzip \ libuuid-devel @@ -299,6 +291,9 @@ jobs: runs-on: ubuntu-24.04 container: almalinux:9 steps: + - name: Checkout installed-consumer fixture + uses: actions/checkout@v7 + - name: Download RPM packages uses: actions/download-artifact@v8 with: @@ -324,25 +319,9 @@ jobs: - name: Build and run an installed-package consumer shell: bash run: | - mkdir -p consumer - cat > consumer/CMakeLists.txt <<'CMAKE' - cmake_minimum_required(VERSION 3.11) - project(TsFileInstalledConsumer LANGUAGES CXX) - - find_package(TsFile CONFIG REQUIRED) - add_executable(installed_consumer main.cpp) - target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) - CMAKE - cat > consumer/main.cpp <<'CPP' - #include - - int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; - } - CPP - cmake -S consumer -B consumer/build + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build cmake --build consumer/build --parallel - ./consumer/build/installed_consumer + ./consumer/build/tsfile_installed_consumer build-windows: name: Build Windows MSVC x86_64 ZIP @@ -370,6 +349,8 @@ jobs: -DBUILD_TOOLS=ON ` -DTSFILE_ENABLE_CPACK=ON ` -DTSFILE_DEPENDENCY_SOURCE=BUNDLED ` + -DTSFILE_MSVC_STATIC_RUNTIME=ON ` + -DCMAKE_PROJECT_TsFile_CPP_INCLUDE="$((Resolve-Path cpp/cmake/tests/CheckStaticMSVCRuntime.cmake).Path)" ` -DTSFILE_ENABLE_NATIVE_ARCH=OFF ` -DTSFILE_ARCHIVE_VERSION="$env:TSFILE_ARCHIVE_VERSION" cmake --build build/windows --config Release --parallel @@ -396,29 +377,16 @@ jobs: } } + python packaging/scripts/verify_windows_runtime.py "$stage" $env:PATH = "$stage/bin;$env:PATH" tsfile-cli.exe --version - New-Item -ItemType Directory -Path consumer -Force | Out-Null - @' - cmake_minimum_required(VERSION 3.11) - project(TsFileInstalledConsumer LANGUAGES CXX) - - find_package(TsFile CONFIG REQUIRED) - add_executable(installed_consumer main.cpp) - target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) - '@ | Set-Content -Path consumer/CMakeLists.txt - @' - #include - - int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; - } - '@ | Set-Content -Path consumer/main.cpp - cmake -S consumer -B consumer/build -G "Visual Studio 17 2022" -A x64 ` + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build ` + -G "Visual Studio 17 2022" -A x64 ` + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ` -DCMAKE_PREFIX_PATH="$stage" cmake --build consumer/build --config Release --parallel - & consumer/build/Release/installed_consumer.exe + & consumer/build/Release/tsfile_installed_consumer.exe - name: Build and inspect combined ZIP env: @@ -469,6 +437,13 @@ jobs: $archive.Dispose() } + Expand-Archive -LiteralPath $archives[0].FullName -DestinationPath extracted/windows + python packaging/scripts/verify_windows_runtime.py extracted/windows + $zipCli = @(Get-ChildItem extracted/windows -Recurse -Filter tsfile-cli.exe -File) + if ($zipCli.Count -ne 1) { throw 'Expected exactly one extracted CLI' } + $env:PATH = "$($zipCli[0].DirectoryName);$env:SystemRoot/System32" + & $zipCli[0].FullName --version + - name: Upload Windows ZIP uses: actions/upload-artifact@v7 with: diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c7b486e00..d7863ba88 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -17,6 +17,18 @@ specific language governing permissions and limitations under the License. ]] cmake_minimum_required(VERSION 3.11) +option(TSFILE_MSVC_STATIC_RUNTIME + "Use the static MSVC runtime throughout a portable bundled build" OFF) +if (TSFILE_MSVC_STATIC_RUNTIME) + if (NOT POLICY CMP0091) + message(FATAL_ERROR "TSFILE_MSVC_STATIC_RUNTIME requires CMake 3.15+") + endif () + # The policy must be selected before the first project(), and inherited by + # upstream projects that reset their policy version in add_subdirectory(). + cmake_policy(SET CMP0091 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0091 NEW) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") +endif () project(TsFile_CPP) if (DEFINED ToolChain) @@ -62,6 +74,12 @@ if (NOT TSFILE_ABI_VERSION MATCHES "^[0-9]+$") endif () include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/DependencySource.cmake) +if (TSFILE_MSVC_STATIC_RUNTIME AND + NOT TSFILE_DEPENDENCY_SOURCE STREQUAL "BUNDLED") + message(FATAL_ERROR + "TSFILE_MSVC_STATIC_RUNTIME requires TSFILE_DEPENDENCY_SOURCE=BUNDLED " + "so prebuilt dependencies cannot introduce a different MSVC runtime") +endif () list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") option(TSFILE_BUILD_SHARED "Build libtsfile as a shared library" ON) @@ -514,6 +532,13 @@ install(FILES COMPONENT development) set(TSFILE_PKGCONFIG_CFLAGS "") +if (NOT TSFILE_BUILD_SHARED) + string(APPEND TSFILE_PKGCONFIG_CFLAGS " -DTSFILE_STATIC") + if (ENABLE_ANTLR4 AND (TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED" OR + TSFILE_ANTLR4_SYSTEM_STATIC)) + string(APPEND TSFILE_PKGCONFIG_CFLAGS " -DANTLR4CPP_STATIC") + endif () +endif () foreach (_TSFILE_PUBLIC_FEATURE ENABLE_ANTLR4 ENABLE_MEM_STAT @@ -531,6 +556,8 @@ foreach (_TSFILE_PUBLIC_FEATURE " -D${_TSFILE_PUBLIC_FEATURE}") endif () endforeach () +file(RELATIVE_PATH TSFILE_PKGCONFIG_PREFIX_RELATIVE + "${CMAKE_INSTALL_FULL_LIBDIR}/pkgconfig" "${CMAKE_INSTALL_PREFIX}") configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libtsfile.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libtsfile.pc" diff --git a/cpp/cmake/TsFileConfig.cmake.in b/cpp/cmake/TsFileConfig.cmake.in index eea711a9e..1357f4898 100644 --- a/cpp/cmake/TsFileConfig.cmake.in +++ b/cpp/cmake/TsFileConfig.cmake.in @@ -24,6 +24,20 @@ if (@ENABLE_THREADS@) find_dependency(Threads) endif() +if (NOT @TSFILE_BUILD_SHARED@) + set(_TSFILE_SAVED_MODULE_PATH "${CMAKE_MODULE_PATH}") + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + file(GLOB _TSFILE_STATIC_CONFIGS + "${CMAKE_CURRENT_LIST_DIR}/TsFileStaticDependencies-*.cmake") + foreach (_TSFILE_STATIC_CONFIG IN LISTS _TSFILE_STATIC_CONFIGS) + include("${_TSFILE_STATIC_CONFIG}") + endforeach () + set(CMAKE_MODULE_PATH "${_TSFILE_SAVED_MODULE_PATH}") + unset(_TSFILE_STATIC_CONFIG) + unset(_TSFILE_STATIC_CONFIGS) + unset(_TSFILE_SAVED_MODULE_PATH) +endif () + include("${CMAKE_CURRENT_LIST_DIR}/TsFileTargets.cmake") check_required_components(TsFile) diff --git a/cpp/cmake/TsFileStaticDependencies.cmake b/cpp/cmake/TsFileStaticDependencies.cmake new file mode 100644 index 000000000..a4942ca77 --- /dev/null +++ b/cpp/cmake/TsFileStaticDependencies.cmake @@ -0,0 +1,104 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# Static archives do not absorb their link dependencies. Keep the private +# build wrappers out of the export: install bundled archives and describe +# imported link-only targets, or rediscover system packages on the consumer. +# install(FILES) also works on CMake 3.11 for targets in another directory. +function(tsfile_install_static_dependency NAME SOURCE SYSTEM_TARGET FIND_CODE) + if (NOT TARGET TsFile::${NAME}) + return() + endif () + target_link_libraries(tsfile INTERFACE + $>) + set(_CODE "${TSFILE_STATIC_DEPENDENCY_CODE}") + if (SOURCE STREQUAL "BUNDLED") + get_target_property(_TARGET TsFile::${NAME} INTERFACE_LINK_LIBRARIES) + install(FILES $ + DESTINATION "${CMAKE_INSTALL_LIBDIR}/tsfile/$" + COMPONENT development) + string(APPEND _CODE + "if (NOT TARGET TsFile::Static_${NAME})\n" + " add_library(TsFile::Static_${NAME} STATIC IMPORTED)\n" + "endif ()\n" + "set_property(TARGET TsFile::Static_${NAME} APPEND PROPERTY IMPORTED_CONFIGURATIONS $>)\n" + "set_target_properties(TsFile::Static_${NAME} PROPERTIES IMPORTED_LOCATION_$> \"\${CMAKE_CURRENT_LIST_DIR}/../../tsfile/$/$\")\n") + else () + string(APPEND _CODE "${FIND_CODE}\n" + "if (NOT TARGET TsFile::Static_${NAME})\n" + " add_library(TsFile::Static_${NAME} INTERFACE IMPORTED)\n" + " set_target_properties(TsFile::Static_${NAME} PROPERTIES INTERFACE_LINK_LIBRARIES ${SYSTEM_TARGET})\n" + "endif ()\n") + endif () + set(TSFILE_STATIC_DEPENDENCY_CODE "${_CODE}" PARENT_SCOPE) +endfunction() + +set(TSFILE_STATIC_DEPENDENCY_CODE "") +tsfile_install_static_dependency(ANTLR4 "${TSFILE_ANTLR4_SOURCE}" + "${TSFILE_ANTLR4_SYSTEM_TARGET}" + "find_package(utf8cpp CONFIG QUIET)\nfind_dependency(antlr4-runtime CONFIG)") +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED") + if (WIN32) + string(APPEND TSFILE_STATIC_DEPENDENCY_CODE + "set_property(TARGET TsFile::Static_ANTLR4 PROPERTY INTERFACE_LINK_LIBRARIES ole32)\n") + else () + if (APPLE) + set(_PLATFORM_LIBRARY CoreFoundation) + else () + set(_PLATFORM_LIBRARY uuid) + endif () + string(APPEND TSFILE_STATIC_DEPENDENCY_CODE + "find_library(_TSFILE_ANTLR4_PLATFORM_LIBRARY ${_PLATFORM_LIBRARY})\n" + "if (NOT _TSFILE_ANTLR4_PLATFORM_LIBRARY)\n" + " set(TsFile_FOUND FALSE)\n" + " set(TsFile_NOT_FOUND_MESSAGE \"Static TsFile needs ${_PLATFORM_LIBRARY}\")\n" + " return()\n" + "endif ()\n" + "set_property(TARGET TsFile::Static_ANTLR4 PROPERTY INTERFACE_LINK_LIBRARIES \"\${_TSFILE_ANTLR4_PLATFORM_LIBRARY}\")\n") + unset(_PLATFORM_LIBRARY) + endif () +endif () +tsfile_install_static_dependency(Snappy "${TSFILE_SNAPPY_SOURCE}" Snappy::snappy + "find_dependency(Snappy ${TSFILE_SNAPPY_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(LZ4 "${TSFILE_LZ4_SOURCE}" LZ4::LZ4 + "find_dependency(LZ4 ${TSFILE_LZ4_MIN_VERSION})") +if (ENABLE_LZ4 AND TSFILE_LZ4_SOURCE STREQUAL "SYSTEM") + install(FILES "${CMAKE_SOURCE_DIR}/cmake/FindLZ4.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile" + COMPONENT development) +endif () +tsfile_install_static_dependency(LZOKAY "${TSFILE_LZOKAY_SOURCE}" lzokay::lzokay + "find_dependency(lzokay ${TSFILE_LZOKAY_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(ZLIB "${TSFILE_ZLIB_SOURCE}" ZLIB::ZLIB + "find_dependency(ZLIB ${TSFILE_ZLIB_MIN_VERSION})") +tsfile_install_static_dependency(ZSTD "${TSFILE_ZSTD_SOURCE}" + "${TSFILE_ZSTD_SYSTEM_TARGET}" + "find_dependency(zstd ${TSFILE_ZSTD_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(LibLZMA "${TSFILE_LIBLZMA_SOURCE}" LibLZMA::LibLZMA + "find_dependency(LibLZMA ${TSFILE_LIBLZMA_MIN_VERSION})\nif (NOT TARGET LibLZMA::LibLZMA)\n add_library(LibLZMA::LibLZMA INTERFACE IMPORTED)\n set_target_properties(LibLZMA::LibLZMA PROPERTIES INTERFACE_LINK_LIBRARIES \"\${LIBLZMA_LIBRARIES}\")\nendif ()") + +configure_file("${CMAKE_SOURCE_DIR}/cmake/TsFileStaticDependencies.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies.cmake.in" @ONLY) +file(GENERATE + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies-$.cmake" + INPUT "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies.cmake.in") +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies-$.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile" + COMPONENT development) +unset(TSFILE_STATIC_DEPENDENCY_CODE) diff --git a/cpp/cmake/TsFileStaticDependencies.cmake.in b/cpp/cmake/TsFileStaticDependencies.cmake.in new file mode 100644 index 000000000..877bc21df --- /dev/null +++ b/cpp/cmake/TsFileStaticDependencies.cmake.in @@ -0,0 +1,20 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +@TSFILE_STATIC_DEPENDENCY_CODE@ diff --git a/cpp/cmake/libtsfile.pc.in b/cpp/cmake/libtsfile.pc.in index ff820d0d8..f7b3db9bf 100644 --- a/cpp/cmake/libtsfile.pc.in +++ b/cpp/cmake/libtsfile.pc.in @@ -18,7 +18,7 @@ # The .pc file is installed below ${prefix}/@CMAKE_INSTALL_LIBDIR@/pkgconfig. # Deriving the prefix from its own location keeps DESTDIR and relocations # working without embedding the build-time install prefix. -prefix=${pcfiledir}/../.. +prefix=${pcfiledir}/@TSFILE_PKGCONFIG_PREFIX_RELATIVE@ exec_prefix=${prefix} libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ diff --git a/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake b/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake new file mode 100644 index 000000000..a30ddfcab --- /dev/null +++ b/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake @@ -0,0 +1,38 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# A configure-time contract test also runnable on non-MSVC hosts. Runtime +# properties initialize on every platform; Windows CI checks actual PE imports. +function(tsfile_check_static_runtime DIRECTORY) + get_property(_TARGETS DIRECTORY "${DIRECTORY}" PROPERTY BUILDSYSTEM_TARGETS) + foreach (_TARGET IN LISTS _TARGETS) + get_target_property(_TYPE ${_TARGET} TYPE) + if (_TYPE MATCHES "^(STATIC_LIBRARY|SHARED_LIBRARY|OBJECT_LIBRARY|EXECUTABLE)$") + get_target_property(_RUNTIME ${_TARGET} MSVC_RUNTIME_LIBRARY) + if (NOT _RUNTIME STREQUAL "MultiThreaded$<$:Debug>") + message(FATAL_ERROR "${_TARGET} has inconsistent MSVC runtime: ${_RUNTIME}") + endif () + endif () + endforeach () + get_property(_SUBDIRECTORIES DIRECTORY "${DIRECTORY}" PROPERTY SUBDIRECTORIES) + foreach (_SUBDIRECTORY IN LISTS _SUBDIRECTORIES) + tsfile_check_static_runtime("${_SUBDIRECTORY}") + endforeach () +endfunction() +cmake_language(DEFER CALL tsfile_check_static_runtime "${CMAKE_SOURCE_DIR}") diff --git a/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt index bf0f83e34..40ea9481d 100644 --- a/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt +++ b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt @@ -17,6 +17,9 @@ specific language governing permissions and limitations under the License. ]] cmake_minimum_required(VERSION 3.11) +if (POLICY CMP0091) + cmake_policy(SET CMP0091 NEW) +endif () project(TsFileInstalledConsumer LANGUAGES CXX) find_package(TsFile CONFIG REQUIRED) diff --git a/cpp/cmake/tests/projects/InstalledConsumer/main.cc b/cpp/cmake/tests/projects/InstalledConsumer/main.cc index 8d242f678..2d265d429 100644 --- a/cpp/cmake/tests/projects/InstalledConsumer/main.cc +++ b/cpp/cmake/tests/projects/InstalledConsumer/main.cc @@ -21,4 +21,7 @@ #include #include -int main() { return TS_DATATYPE_INT32 == 1 ? 0 : 1; } +int main() { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.c b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c index f32e5a550..05a1566f7 100644 --- a/cpp/cmake/tests/projects/PkgConfigConsumer/main.c +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c @@ -18,4 +18,7 @@ */ #include -int main(void) { return TS_DATATYPE_INT32 == 1 ? 0 : 1; } +int main(void) { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc index be94b8543..daa807236 100644 --- a/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc @@ -16,6 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +#include #include -int main() { return 0; } +int main() { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 8854e5d6c..96c7d9f47 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -126,6 +126,10 @@ else() # Consumers of the CMake target must see TSFILE_API without DLL import # decoration when linking the static library on MSVC. target_compile_definitions(tsfile INTERFACE TSFILE_STATIC) + if (ENABLE_ANTLR4 AND (TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED" OR + TSFILE_ANTLR4_SYSTEM_STATIC)) + target_compile_definitions(tsfile INTERFACE ANTLR4CPP_STATIC) + endif () endif() if (${COV_ENABLED}) @@ -151,9 +155,10 @@ foreach (_TSFILE_OBJECT_TARGET IN LISTS _TSFILE_OBJECT_TARGETS) $) endforeach () -if (NOT "${_TSFILE_PROJECT_DEPENDENCIES}" STREQUAL "") - target_link_libraries(tsfile PRIVATE ${_TSFILE_PROJECT_DEPENDENCIES}) -endif () +foreach (_TSFILE_DEPENDENCY IN LISTS _TSFILE_PROJECT_DEPENDENCIES) + target_link_libraries(tsfile PRIVATE $) +endforeach () +unset(_TSFILE_DEPENDENCY) unset(_TSFILE_OBJECT_SOURCES) unset(_TSFILE_OBJECT_TARGET) @@ -230,6 +235,8 @@ if (TSFILE_BUILD_SHARED) # dependencies at link time. Do not export those build-only targets as # consumer link requirements; they are not installed with the package. set_property(TARGET tsfile PROPERTY INTERFACE_LINK_LIBRARIES "") +else () + include(${CMAKE_SOURCE_DIR}/cmake/TsFileStaticDependencies.cmake) endif () # A shared library is a RUNTIME plus an import ARCHIVE on Windows and a LIBRARY diff --git a/packaging/README.md b/packaging/README.md index 910026236..feffc2cb2 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -147,11 +147,52 @@ runtime, development files, and tools under a relocatable prefix. In particular, the MSVC outputs are installed as `bin/tsfile.dll`, `lib/tsfile.lib`, and `bin/tsfile-cli.exe`. +The portable ZIP targets Windows 10 / Windows Server 2022 or newer and uses +the static MSVC runtime (`/MT`) in Release. `TSFILE_MSVC_STATIC_RUNTIME=ON` +requires CMake 3.15+ and `TSFILE_DEPENDENCY_SOURCE=BUNDLED`: all codec archives, +TsFile, and the CLI are compiled with the same runtime selection. The workflow +checks the runtime property on every native target and inspects every shipped +EXE/DLL with `dumpbin`, rejecting dynamic Visual C++ runtime imports and missing +non-system DLLs in both the staged tree and the extracted ZIP. Running the CLI +does not require a separate Visual C++ Redistributable installation. Microsoft +redistributable DLLs are not copied from the runner or included in the ZIP; +runtime security updates require rebuilding the static-runtime package. + +C++ SDK consumers should use the matching MSVC v143 Release toolset and `/MT` +(`CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded`, with CMake policy CMP0091 set to +NEW before `project()`). Keep allocation and release paired through the public +APIs; CRT-owned objects such as `FILE*` must not cross DLL boundaries. The +installed-consumer fixture demonstrates a public setting/getter round trip +that requires real symbols from `tsfile.dll`. + The workflow uploads that ZIP as the intermediate artifact `native-windows-msvc-x86_64` for 14 days. Like the Linux jobs, it validates the license files, CMake package config, staged executable, library, import library, and public headers without publishing the artifact. +## Static SDK and install regressions + +`TSFILE_BUILD_SHARED=OFF` remains installable. Its CMake package rediscovers +system codec packages on the consuming machine and installs bundled codec +archives under `/tsfile/`. Link with `TsFile::tsfile` +to receive the complete static dependency list. Bundled archives and CMake +metadata move with the install prefix; system dependency development packages +must be available on the consuming machine. The shared package retains its +existing runtime/development/tools layout. + +The pkg-config prefix is derived from the configured pkg-config installation +directory, including multiarch library directories. The optional Unix native +integration suite builds, installs, relocates, and links CMake and pkg-config +consumers. It requires CMake 3.19+ for the runtime-property test, pkg-config, +system LZ4, and a verified dependency cache containing ANTLR4, utf8cpp, zlib, +and LZOKAY archives: + +```bash +TSFILE_RUN_INSTALL_TESTS=1 \ +TSFILE_TEST_DEPENDENCY_CACHE=/path/to/dependency-cache \ +python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v +``` + ## Final native package bundle Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM diff --git a/packaging/homebrew/tsfile-dev.rb.in b/packaging/homebrew/tsfile-dev.rb.in index cc5b60bb2..567a2fcfc 100644 --- a/packaging/homebrew/tsfile-dev.rb.in +++ b/packaging/homebrew/tsfile-dev.rb.in @@ -54,7 +54,8 @@ class TsfileDev < Formula #include int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; } CPP diff --git a/packaging/homebrew/tsfile.rb b/packaging/homebrew/tsfile.rb index 3e96f3dca..1beaac75b 100644 --- a/packaging/homebrew/tsfile.rb +++ b/packaging/homebrew/tsfile.rb @@ -59,7 +59,8 @@ def install #include int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; } CPP diff --git a/packaging/scripts/native_package_versions.py b/packaging/scripts/native_package_versions.py index d4bd653eb..19981e377 100644 --- a/packaging/scripts/native_package_versions.py +++ b/packaging/scripts/native_package_versions.py @@ -119,9 +119,16 @@ def main(argv: Sequence[str] | None = None) -> int: ) arguments.json_out.write_text(json.dumps(versions, indent=2) + "\n", encoding="utf-8") if arguments.github_output is not None: - with arguments.github_output.open("a", encoding="utf-8") as github_output: + with arguments.github_output.open("a+b") as github_output: + if github_output.tell(): + github_output.seek(-1, 2) + if github_output.read(1) != b"\n": + github_output.write(b"\n") github_output.write( - "\n".join(f"{key}={value}" for key, value in versions.items()) + "\n" + ( + "\n".join(f"{key}={value}" for key, value in versions.items()) + + "\n" + ).encode("utf-8") ) return 0 diff --git a/packaging/scripts/verify_windows_runtime.py b/packaging/scripts/verify_windows_runtime.py new file mode 100644 index 000000000..79a7c90b4 --- /dev/null +++ b/packaging/scripts/verify_windows_runtime.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reject portable Windows binaries that need an unshipped DLL or dynamic CRT.""" + +import argparse +import re +import subprocess +from pathlib import Path + +# Windows 10 / Server 2022 inbox DLLs used by TsFile and the static CRT. This +# explicit list deliberately does not trust whatever happens to be on PATH or +# in the development runner's System32 (which also contains the VC redist). +SYSTEM_DLLS = { + "advapi32.dll", + "bcrypt.dll", + "crypt32.dll", + "kernel32.dll", + "ntdll.dll", + "ole32.dll", + "oleaut32.dll", + "rpcrt4.dll", + "secur32.dll", + "shell32.dll", + "shlwapi.dll", + "user32.dll", + "version.dll", + "ws2_32.dll", +} +DYNAMIC_CRT = re.compile(r"^(msvcp|msvcr|vcruntime|ucrtbase|api-ms-win-crt-)", re.I) + + +def parse_dependents(output: str) -> set[str]: + imports = set(re.findall(r"^\s+([A-Za-z0-9_.-]+\.dll)\s*$", output, re.M | re.I)) + if not imports: + raise ValueError("dumpbin did not report any DLL imports") + return {name.lower() for name in imports} + + +def validate_dependents(binary: str, imports: set[str], colocated: set[str]) -> None: + for name in sorted(imports): + if DYNAMIC_CRT.match(name): + raise ValueError( + f"{binary} imports {name}; the ZIP requires a static MSVC runtime" + ) + if ( + name not in SYSTEM_DLLS + and not name.startswith("api-ms-win-") + and name not in colocated + ): + raise ValueError(f"{binary} needs an unshipped non-system DLL: {name}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("prefix", type=Path) + args = parser.parse_args() + binaries = sorted( + path + for path in args.prefix.rglob("*") + if path.suffix.lower() in (".exe", ".dll") + ) + if not binaries: + parser.error("the prefix contains no Windows binaries") + for binary in binaries: + output = subprocess.run( + ["dumpbin", "/nologo", "/dependents", str(binary)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ).stdout + imports = parse_dependents(output) + colocated = { + path.name.lower() + for path in binary.parent.iterdir() + if path.suffix.lower() == ".dll" + } + validate_dependents(binary.name, imports, colocated) + print( + f"{binary.name}: static CRT; all {len(imports)} DLL dependencies are inbox or colocated" + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb new file mode 100644 index 000000000..82e5e506f --- /dev/null +++ b/packaging/tests/check_native_workflow.rb @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "yaml" +require "shellwords" +require "open3" + +path = ARGV.fetch(0, File.expand_path("../../.github/workflows/native-packages.yml", __dir__)) +workflow = YAML.load_file(path) +triggers = workflow["on"] || workflow[true] # Psych's YAML 1.1 boolean parsing +raise "Workflow must remain manual-only" unless triggers.keys == ["workflow_dispatch"] +raise "Workflow permissions must remain read-only" unless workflow["permissions"] == { "contents" => "read" } + +jobs = workflow.fetch("jobs") +bootstrap = jobs.fetch("build-rpm").fetch("steps").find { |step| step["name"] == "Install RPM build prerequisites" } +packages = Shellwords.split(bootstrap.fetch("run").gsub("\\\n", " ")) +raise "AlmaLinux 9 curl-minimal conflicts with full curl" if packages.include?("curl") +raise "RPM bootstrap must retain curl-minimal" unless packages.include?("curl-minimal") + +windows = jobs.fetch("build-windows").fetch("steps").filter_map { |step| step["run"] }.join("\n") +raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-DTSFILE_MSVC_STATIC_RUNTIME=ON") && windows.include?("-DTSFILE_DEPENDENCY_SOURCE=BUNDLED") +raise "All native target runtimes must be checked" unless windows.include?("CheckStaticMSVCRuntime.cmake") +raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 + +%w[test-deb test-rpm build-windows].each do |name| + steps = jobs.fetch(name).fetch("steps") + raise "#{name} must check out the consumer fixture" unless steps.any? { |step| step["uses"].to_s.start_with?("actions/checkout@") } + runs = steps.filter_map { |step| step["run"] }.join("\n") + raise "#{name} must use the installed CMake consumer" unless runs.include?("-S cpp/cmake/tests/projects/InstalledConsumer") +end + +jobs.each_value do |job| + job.fetch("steps").each do |step| + next unless step["run"] && step.fetch("shell", "bash") == "bash" + output, status = Open3.capture2e("bash", "-n", stdin_data: step["run"]) + raise "Invalid shell in #{step['name']}: #{output}" unless status.success? + end +end +puts "PASS: manual/read-only workflow, AlmaLinux bootstrap, Windows CRT checks, shared consumers, and Bash syntax" diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py new file mode 100644 index 000000000..ea0574834 --- /dev/null +++ b/packaging/tests/test_native_install.py @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Opt-in native configure/install/link regressions (requires CMake and LZ4).""" + +import os +import re +import shlex +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +FIXTURES = ROOT / "cpp/cmake/tests/projects" + + +@unittest.skipUnless( + os.environ.get("TSFILE_RUN_INSTALL_TESTS") == "1", "opt-in native builds" +) +class NativeInstallTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="tsfile-install-") + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.build = self.directory / "build" + self.stage = self.directory / "stage" + + def run_command(self, *arguments, env=None): + result = subprocess.run( + [str(argument) for argument in arguments], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + ) + self.assertEqual(result.returncode, 0, result.stdout[-14000:]) + return result.stdout.strip() + + def configure(self, *options): + arguments = [ + "cmake", + "-S", + ROOT / "cpp", + "-B", + self.build, + "-DCMAKE_BUILD_TYPE=Release", + "-DBUILD_TEST=OFF", + "-DBUILD_TOOLS=OFF", + "-DTSFILE_DEPENDENCY_SOURCE=SYSTEM", + "-DTSFILE_ENABLE_NATIVE_ARCH=OFF", + f"-DCMAKE_INSTALL_PREFIX={self.stage}", + ] + arguments.extend( + f"-DENABLE_{name}=OFF" + for name in ( + "ANTLR4", + "SNAPPY", + "LZ4", + "LZOKAY", + "ZLIB", + "ZSTD", + "LZMA2", + "SIMD", + ) + ) + self.run_command(*arguments, *options) + + def install(self): + self.run_command("cmake", "--build", self.build, "--parallel", "8") + self.run_command("cmake", "--install", self.build, "--config", "Release") + relocated = self.directory / "relocated" + self.stage.rename(relocated) + self.stage = relocated + for path in self.stage.rglob("*.cmake"): + self.assertNotIn(str(self.build), path.read_text(), str(path)) + self.assertNotIn(str(ROOT / "cpp"), path.read_text(), str(path)) + + def consumer(self, fixture, target, env=None, *options): + build = self.directory / fixture + self.run_command( + "cmake", + "-S", + FIXTURES / fixture, + "-B", + build, + f"-DCMAKE_PREFIX_PATH={self.stage}", + *options, + env=env, + ) + self.run_command("cmake", "--build", build, "--parallel", "8", env=env) + self.run_command(build / target, env=env) + + def test_static_system_lz4_install_and_relocated_consumer(self): + self.configure("-DTSFILE_BUILD_SHARED=OFF", "-DENABLE_LZ4=ON") + self.install() + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + + def test_static_bundled_install_and_relocated_consumer(self): + self.configure( + "-DTSFILE_BUILD_SHARED=OFF", + "-DTSFILE_DEPENDENCY_SOURCE=BUNDLED", + "-DENABLE_ANTLR4=ON", + "-DENABLE_ZLIB=ON", + "-DENABLE_LZOKAY=ON", + "-DTSFILE_DEPENDENCY_OFFLINE=ON", + f"-DTSFILE_DEPENDENCY_CACHE={os.environ['TSFILE_TEST_DEPENDENCY_CACHE']}", + ) + self.install() + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + + def test_multiarch_pkgconfig_install_and_relocated_consumers(self): + self.configure("-DCMAKE_INSTALL_LIBDIR=lib/x86_64-linux-gnu") + self.install() + env = dict( + os.environ, + PKG_CONFIG_PATH=str(self.stage / "lib/x86_64-linux-gnu/pkgconfig"), + ) + prefix = self.run_command( + "pkg-config", "--variable=prefix", "libtsfile", env=env + ) + self.assertEqual(Path(prefix).resolve(), self.stage.resolve()) + self.consumer("PkgConfigConsumer", "tsfile_pkgconfig_consumer", env) + self.run_command( + self.directory / "PkgConfigConsumer/tsfile_pkgconfig_cpp_consumer", env=env + ) + + def test_installed_smoke_requires_a_library_symbol(self): + self.configure() + self.install() + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-std=c++11", + f"-I{self.stage / 'include'}", + f"-I{self.stage / 'include/tsfile'}", + str(FIXTURES / "InstalledConsumer/main.cc"), + "-o", + str(self.directory / "unlinked"), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertNotEqual( + result.returncode, 0, "Smoke consumer linked successfully without libtsfile" + ) + self.assertIn("get_global_time_encoding", result.stdout) + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + for formula in ("tsfile.rb", "tsfile-dev.rb.in"): + source = re.search( + r"write <<~CPP\n(.*?)\n\s+CPP", + (ROOT / "packaging/homebrew" / formula).read_text(), + re.S, + ).group(1) + source_path = self.directory / f"{formula}.cc" + source_path.write_text(textwrap.dedent(source)) + command = [ + *shlex.split(os.environ.get("CXX", "c++")), + "-std=c++11", + f"-I{self.stage / 'include'}", + str(source_path), + "-o", + str(self.directory / "brew-consumer"), + ] + unlinked = subprocess.run( + command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + self.assertNotEqual(unlinked.returncode, 0, formula) + self.assertIn("get_global_time_encoding", unlinked.stdout) + self.run_command( + *command, + f"-L{self.stage / 'lib'}", + f"-Wl,-rpath,{self.stage / 'lib'}", + "-ltsfile", + ) + self.run_command(self.directory / "brew-consumer") + + def test_static_msvc_runtime_reaches_dependency_and_project_targets(self): + self.configure( + "-DTSFILE_MSVC_STATIC_RUNTIME=ON", + "-DTSFILE_DEPENDENCY_SOURCE=BUNDLED", + "-DENABLE_ZLIB=ON", + "-DTSFILE_DEPENDENCY_OFFLINE=ON", + f"-DTSFILE_DEPENDENCY_CACHE={os.environ['TSFILE_TEST_DEPENDENCY_CACHE']}", + f"-DCMAKE_PROJECT_TsFile_CPP_INCLUDE={ROOT / 'cpp/cmake/tests/CheckStaticMSVCRuntime.cmake'}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py index a2cc05221..b5a6b80c9 100644 --- a/packaging/tests/test_native_package_versions.py +++ b/packaging/tests/test_native_package_versions.py @@ -141,6 +141,47 @@ def test_cli_writes_json_and_github_outputs(self): ["existing=value", *[f"{key}={value}" for key, value in expected.items()]], ) + def test_cli_separates_unterminated_github_output_without_changing_bytes(self): + for existing in ( + b"", + b"existing=value", + b"existing=value\n", + b"existing=value\r\n", + ): + with self.subTest( + existing=existing + ), tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "github-output.txt" + output.write_bytes(existing) + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--cmake-file", + str(CPP_CMAKE_FILE), + "--build-date", + "20260910", + "--run-number", + "123", + "--run-attempt", + "1", + "--git-sha", + "abcdef123456", + "--json-out", + str(Path(directory) / "versions.json"), + "--github-output", + str(output), + ], + check=True, + ) + separator = b"\n" if existing and not existing.endswith(b"\n") else b"" + self.assertTrue( + output.read_bytes().startswith( + existing + separator + b"base_version=2.5.0\n" + ), + output.read_bytes(), + ) + if __name__ == "__main__": unittest.main() diff --git a/packaging/tests/test_windows_runtime.py b/packaging/tests/test_windows_runtime.py new file mode 100644 index 000000000..8258f00a0 --- /dev/null +++ b/packaging/tests/test_windows_runtime.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts/verify_windows_runtime.py" + + +class WindowsRuntimeTest(unittest.TestCase): + def checker(self): + self.assertTrue( + MODULE_PATH.is_file(), "Windows artifacts need a runtime dependency check" + ) + spec = importlib.util.spec_from_file_location( + "verify_windows_runtime", MODULE_PATH + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_allows_only_inbox_or_colocated_package_dlls(self): + module = self.checker() + imports = module.parse_dependents( + "File Type: EXECUTABLE IMAGE\n\n Image has the following dependencies:\n\n" + " tsfile.dll\n KERNEL32.dll\n api-ms-win-core-synch-l1-2-0.dll\n\n" + " Summary\n 1000 .data\n" + ) + module.validate_dependents("tsfile-cli.exe", imports, {"tsfile.dll"}) + self.assertEqual(len(imports), 3) + + def test_rejects_dynamic_crt_even_when_runner_or_package_provides_it(self): + module = self.checker() + for dll in ( + "VCRUNTIME140.dll", + "MSVCP140.dll", + "ucrtbase.dll", + "api-ms-win-crt-runtime-l1-1-0.dll", + ): + with self.subTest(dll=dll), self.assertRaisesRegex( + ValueError, "static MSVC runtime" + ): + module.validate_dependents("tsfile.dll", {dll.lower()}, {dll.lower()}) + + def test_rejects_unshipped_codec_dll(self): + module = self.checker() + with self.assertRaisesRegex(ValueError, "snappy.dll"): + module.validate_dependents("tsfile.dll", {"snappy.dll"}, {"tsfile.dll"}) + + def test_rejects_unrecognized_dumpbin_output(self): + with self.assertRaisesRegex(ValueError, "DLL imports"): + self.checker().parse_dependents( + "fatal error LNK1107: invalid or corrupt file" + ) + + +if __name__ == "__main__": + unittest.main() From d3bd297b2b7116314b26986a2806d7945a730e37 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 22:35:05 +0800 Subject: [PATCH 16/27] fix(cmake): preserve static SDK ANTLR compatibility bounds --- cpp/cmake/TsFileStaticDependencies.cmake | 26 +++- packaging/tests/test_native_install.py | 163 +++++++++++++++++++++++ 2 files changed, 188 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/TsFileStaticDependencies.cmake b/cpp/cmake/TsFileStaticDependencies.cmake index a4942ca77..8233487f3 100644 --- a/cpp/cmake/TsFileStaticDependencies.cmake +++ b/cpp/cmake/TsFileStaticDependencies.cmake @@ -50,9 +50,33 @@ function(tsfile_install_static_dependency NAME SOURCE SYSTEM_TARGET FIND_CODE) endfunction() set(TSFILE_STATIC_DEPENDENCY_CODE "") +# Keep the build-side compatibility interval when a static consumer rediscovers +# ANTLR4. Explicit comparisons also support CMake 3.11 and packages that report +# ANTLR_VERSION instead of antlr4-runtime_VERSION. +string(CONFIGURE [=[ +find_package(utf8cpp CONFIG QUIET) +find_package(antlr4-runtime CONFIG QUIET) +set(_TSFILE_ANTLR4_VERSION "${antlr4-runtime_VERSION}") +if ("${_TSFILE_ANTLR4_VERSION}" STREQUAL "" AND DEFINED ANTLR_VERSION) + set(_TSFILE_ANTLR4_VERSION "${ANTLR_VERSION}") +endif () +if (NOT antlr4-runtime_FOUND OR + "${_TSFILE_ANTLR4_VERSION}" STREQUAL "" OR + _TSFILE_ANTLR4_VERSION VERSION_LESS "@TSFILE_ANTLR4_MIN_VERSION@" OR + NOT _TSFILE_ANTLR4_VERSION VERSION_LESS "@TSFILE_ANTLR4_NEXT_INCOMPATIBLE_VERSION@" OR + NOT TARGET @TSFILE_ANTLR4_SYSTEM_TARGET@) + set(TsFile_FOUND FALSE) + set(TsFile_NOT_FOUND_MESSAGE + "Static TsFile requires a compatible system ANTLR4 >=@TSFILE_ANTLR4_MIN_VERSION@ and <@TSFILE_ANTLR4_NEXT_INCOMPATIBLE_VERSION@ with target @TSFILE_ANTLR4_SYSTEM_TARGET@; reported version '${_TSFILE_ANTLR4_VERSION}'.") + unset(_TSFILE_ANTLR4_VERSION) + return() +endif () +unset(_TSFILE_ANTLR4_VERSION) +]=] _TSFILE_FIND_STATIC_ANTLR4 @ONLY) tsfile_install_static_dependency(ANTLR4 "${TSFILE_ANTLR4_SOURCE}" "${TSFILE_ANTLR4_SYSTEM_TARGET}" - "find_package(utf8cpp CONFIG QUIET)\nfind_dependency(antlr4-runtime CONFIG)") + "${_TSFILE_FIND_STATIC_ANTLR4}") +unset(_TSFILE_FIND_STATIC_ANTLR4) if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED") if (WIN32) string(APPEND TSFILE_STATIC_DEPENDENCY_CODE diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py index ea0574834..76128fcf9 100644 --- a/packaging/tests/test_native_install.py +++ b/packaging/tests/test_native_install.py @@ -20,6 +20,7 @@ import os import re import shlex +import shutil import subprocess import tempfile import textwrap @@ -30,6 +31,168 @@ FIXTURES = ROOT / "cpp/cmake/tests/projects" +class StaticANTLRPackageTest(unittest.TestCase): + """Exercise installed metadata without compiling the ANTLR runtime.""" + + def test_installed_system_antlr_requires_compatible_runtime(self): + with tempfile.TemporaryDirectory(prefix="tsfile-antlr-config-") as temporary: + directory = Path(temporary) + producer = directory / "producer" + (producer / "cmake").mkdir(parents=True) + shutil.copyfile( + ROOT / "cpp/cmake/TsFileStaticDependencies.cmake.in", + producer / "cmake/TsFileStaticDependencies.cmake.in", + ) + (producer / "CMakeLists.txt").write_text(textwrap.dedent("""\ + cmake_minimum_required(VERSION 3.11) + project(StaticANTLRPackage NONE) + include(CMakePackageConfigHelpers) + include("${TSFILE_CMAKE_DIR}/DependencySource.cmake") + include("${TSFILE_CMAKE_DIR}/ANTLR4Dependency.cmake") + set(CMAKE_INSTALL_LIBDIR lib) + set(ENABLE_ANTLR4 ON) + set(ENABLE_THREADS OFF) + set(TSFILE_BUILD_SHARED OFF) + add_library(tsfile INTERFACE) + add_library(TsFile::ANTLR4 INTERFACE IMPORTED) + include("${TSFILE_CMAKE_DIR}/TsFileStaticDependencies.cmake") + configure_package_config_file( + "${TSFILE_CMAKE_DIR}/TsFileConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + INSTALL_DESTINATION lib/cmake/TsFile) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + DESTINATION lib/cmake/TsFile) + install(TARGETS tsfile EXPORT TsFileTargets) + install(EXPORT TsFileTargets NAMESPACE TsFile:: + DESTINATION lib/cmake/TsFile) + """)) + consumer = directory / "consumer" + consumer.mkdir() + (consumer / "CMakeLists.txt").write_text(textwrap.dedent("""\ + cmake_minimum_required(VERSION 3.11) + project(StaticANTLRConsumer NONE) + find_package(TsFile CONFIG REQUIRED) + get_target_property(runtime TsFile::Static_ANTLR4 + INTERFACE_LINK_LIBRARIES) + if (NOT runtime STREQUAL EXPECTED_TARGET OR NOT TARGET ${runtime}) + message(FATAL_ERROR "Static TsFile lost its system target") + endif () + if (NOT TARGET utf8cpp) + message(FATAL_ERROR "Static TsFile lost utf8cpp compatibility") + endif () + """)) + + def configure(source, build, *options): + return subprocess.run( + [ + "cmake", + "-S", + str(source), + "-B", + str(build), + "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=FALSE", + "-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=FALSE", + "-DCMAKE_FIND_USE_CMAKE_SYSTEM_PATH=FALSE", + "-DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=FALSE", + f"-DCMAKE_MAKE_PROGRAM={shutil.which('make')}", + "-G", + "Unix Makefiles", + *options, + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + def package(prefix, version, target, version_field): + prefix.mkdir(parents=True) + (prefix / "utf8cppConfig.cmake").write_text( + "add_library(utf8cpp INTERFACE IMPORTED)\n" + ) + if version == "missing-package": + return + content = "" + if version is not None: + content += f'set({version_field} "{version}")\n' + if target is not None: + content += ( + f"add_library({target} INTERFACE IMPORTED)\n" + f"set_property(TARGET {target} PROPERTY " + "INTERFACE_LINK_LIBRARIES utf8cpp)\n" + ) + (prefix / "antlr4-runtime-config.cmake").write_text(content) + + targets = ( + "antlr4_static", + "antlr4-runtime::antlr4_static", + "antlr4_shared", + "antlr4-runtime::antlr4_shared", + ) + for index, target in enumerate(targets): + build = directory / f"producer-build-{index}" + stage = directory / f"stage-{index}" + build_package = directory / f"build-package-{index}" + package(build_package, "4.9.3", target, "ANTLR_VERSION") + result = configure( + producer, + build, + f"-DTSFILE_CMAKE_DIR={ROOT / 'cpp/cmake'}", + "-DTSFILE_DEPENDENCY_SOURCE=SYSTEM", + f"-DCMAKE_PREFIX_PATH={build_package}", + f"-DCMAKE_INSTALL_PREFIX={stage}", + ) + self.assertEqual(result.returncode, 0, result.stdout) + result = subprocess.run( + ["cmake", "--install", str(build)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + relocated = directory / f"relocated-{index}" + stage.rename(relocated) + cases = ( + ("4.9.3", target, True), + ("4.12.0", target, True), + ("4.13.0", target, False), + ("4.13.2", target, False), + ("4.9.2", target, False), + (None, target, False), + ("4.9.3", None, False), + ("missing-package", None, False), + ) + for field in ("antlr4-runtime_VERSION", "ANTLR_VERSION"): + for case, (version, consumer_target, succeeds) in enumerate(cases): + with self.subTest( + target=target, + field=field, + version=version, + consumer_target=consumer_target, + ): + suffix = f"{index}-{field}-{case}" + runtime = directory / f"runtime-{suffix}" + package(runtime, version, consumer_target, field) + result = configure( + consumer, + directory / f"consumer-{suffix}", + f"-DCMAKE_PREFIX_PATH={relocated};{runtime}", + f"-DEXPECTED_TARGET={target}", + ) + if succeeds: + self.assertEqual(result.returncode, 0, result.stdout) + else: + self.assertNotEqual( + result.returncode, + 0, + f"Installed static TsFile accepted {version}:\n" + + result.stdout, + ) + diagnostic = " ".join(result.stdout.split()) + self.assertIn("compatible system ANTLR4", diagnostic) + self.assertIn(">=4.9.3", diagnostic) + self.assertIn("<4.13.0", diagnostic) + + @unittest.skipUnless( os.environ.get("TSFILE_RUN_INSTALL_TESTS") == "1", "opt-in native builds" ) From 485b99f43c4e1aa4303f2e7630f5f4182d6becb8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 23:18:44 +0800 Subject: [PATCH 17/27] fix(ci): address native packaging runner failures --- .github/workflows/native-packages.yml | 4 +-- cpp/src/utils/util_define.h | 4 +-- packaging/README.md | 7 ++++- packaging/homebrew/tsfile-dev.rb.in | 1 + packaging/tests/check_native_workflow.rb | 5 ++++ packaging/tests/test_native_install.py | 30 +++++++++++++++++++ .../tests/test_render_homebrew_formula.py | 5 ++++ 7 files changed, 51 insertions(+), 5 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index f6f349315..99bf2d8bb 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -205,7 +205,7 @@ jobs: dnf install -y \ cmake \ gcc-c++ \ - ninja-build \ + make \ pkgconf-pkg-config \ rpm-build \ git \ @@ -224,7 +224,7 @@ jobs: TSFILE_RPM_PACKAGE_RELEASE: ${{ needs.prepare.outputs.rpm_release }} shell: bash run: | - cmake -S cpp -B build/rpm -G Ninja \ + cmake -S cpp -B build/rpm \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_TEST=OFF \ -DBUILD_TOOLS=ON \ diff --git a/cpp/src/utils/util_define.h b/cpp/src/utils/util_define.h index 564581a11..f7f638e5b 100644 --- a/cpp/src/utils/util_define.h +++ b/cpp/src/utils/util_define.h @@ -147,13 +147,13 @@ typedef int mode_t; * @msg should be a single word (use -/_ to concat) * such as This_should_be_TRUE */ -#if __cplusplus < 201103L +#if __cplusplus < 201103L && !defined(_MSC_VER) // TODO only define this when DEBUG #define STATIC_ASSERT(cond, msg) \ typedef char static_assertion_##msg[(cond) ? 1 : -1] __attribute__((unused)) #else #define STATIC_ASSERT(cond, msg) static_assert((cond), #msg) -#endif // __cplusplus < 201103L +#endif // __cplusplus < 201103L && !defined(_MSC_VER) /* ======== atomic operation ======== * diff --git a/packaging/README.md b/packaging/README.md index feffc2cb2..32b8106d4 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -117,7 +117,12 @@ Formula `tsfile-dev`, using `packaging/homebrew/tsfile-dev.rb.in`. This is not a Homebrew/core submission. Each build pins the full workflow commit from `ColinLeeo/tsfile`, hashes that source archive, and uses the generated immutable Homebrew development version. The Formula retains the CMake install behavior -and tests the installed C++ consumer and CLI before bottling. +and tests the installed C++ consumer and CLI before bottling. `tsfile-dev` is +keg-only because the current SDK intentionally ships its dependency header +closure; linking that closure into Homebrew's shared prefix would collide with +headers owned by dependencies such as `simde`. After installation, invoke the +CLI as `$(brew --prefix tsfile-dev)/bin/tsfile-cli`, or add that Formula's +`bin` directory to `PATH`. The ARM64 job runs on `macos-latest` and the Intel job on `macos-15-intel`. Their `native-homebrew-macos-arm64` and `native-homebrew-macos-x86_64` diff --git a/packaging/homebrew/tsfile-dev.rb.in b/packaging/homebrew/tsfile-dev.rb.in index 567a2fcfc..b5eac6431 100644 --- a/packaging/homebrew/tsfile-dev.rb.in +++ b/packaging/homebrew/tsfile-dev.rb.in @@ -22,6 +22,7 @@ class TsfileDev < Formula version "@HOMEBREW_VERSION@" sha256 "@SOURCE_SHA256@" license "Apache-2.0" + keg_only "development snapshots install their dependency header closure" @BOTTLE_BLOCK@ diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 82e5e506f..053e25174 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -30,6 +30,11 @@ packages = Shellwords.split(bootstrap.fetch("run").gsub("\\\n", " ")) raise "AlmaLinux 9 curl-minimal conflicts with full curl" if packages.include?("curl") raise "RPM bootstrap must retain curl-minimal" unless packages.include?("curl-minimal") +raise "AlmaLinux 9 default repositories do not provide ninja-build" if packages.include?("ninja-build") +raise "RPM bootstrap must install make" unless packages.include?("make") + +rpm_build = jobs.fetch("build-rpm").fetch("steps").filter_map { |step| step["run"] }.join("\n") +raise "RPM build must not require Ninja" if rpm_build.include?("-G Ninja") windows = jobs.fetch("build-windows").fetch("steps").filter_map { |step| step["run"] }.join("\n") raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-DTSFILE_MSVC_STATIC_RUNTIME=ON") && windows.include?("-DTSFILE_DEPENDENCY_SOURCE=BUNDLED") diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py index 76128fcf9..ed3eb5a10 100644 --- a/packaging/tests/test_native_install.py +++ b/packaging/tests/test_native_install.py @@ -31,6 +31,36 @@ FIXTURES = ROOT / "cpp/cmake/tests/projects" +class PublicHeaderCompatibilityTest(unittest.TestCase): + def test_static_assert_uses_the_native_keyword_for_msvc(self): + source = textwrap.dedent("""\ + #include "utils/util_define.h" + class Probe { STATIC_ASSERT(true, MSVC_old_cplusplus); }; + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-E", + "-P", + "-x", + "c++", + "-std=c++11", + "-D_MSC_VER=1930", + "-DTSFILE_STATIC", + "-D__cplusplus=199711L", + f"-I{ROOT / 'cpp/src'}", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout[-4000:]) + self.assertRegex(result.stdout, r"class Probe \{\s*(?:static_assert|_Static_assert)") + self.assertNotIn("static_assertion_MSVC_old_cplusplus", result.stdout) + + class StaticANTLRPackageTest(unittest.TestCase): """Exercise installed metadata without compiling the ANTLR runtime.""" diff --git a/packaging/tests/test_render_homebrew_formula.py b/packaging/tests/test_render_homebrew_formula.py index a702c40d9..a60f9a6c0 100644 --- a/packaging/tests/test_render_homebrew_formula.py +++ b/packaging/tests/test_render_homebrew_formula.py @@ -128,6 +128,11 @@ def test_preserves_ruby_interpolation_and_other_at_signs(self): def test_cli_renders_real_template_with_optional_bottle_file(self): self.renderer() + template = TEMPLATE_PATH.read_text(encoding="utf-8") + self.assertIn( + 'keg_only "development snapshots install their dependency header closure"', + template, + ) with tempfile.TemporaryDirectory() as temporary_directory: directory = Path(temporary_directory) output = directory / "tsfile-dev.rb" From 5094e29c8a50d2e8bb4d7f47e8120009bfe3a972 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 23:55:27 +0800 Subject: [PATCH 18/27] fix(ci): resolve native package portability failures --- .github/workflows/native-packages.yml | 1 + cpp/src/utils/util_define.h | 4 +- packaging/tests/check_native_workflow.rb | 5 ++ packaging/tests/test_native_install.py | 76 +++++++++++++++++++++++- 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 99bf2d8bb..187bb91fd 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -581,6 +581,7 @@ jobs: - name: Merge both bottle tags into the Formula shell: bash run: | + brew trust apache/tsfile-dev brew bottle --merge --write --no-commit homebrew/bottles/*.json tap_path="$(brew --repository apache/tsfile-dev)" cp "$tap_path/Formula/tsfile-dev.rb" homebrew/Formula/tsfile-dev.rb diff --git a/cpp/src/utils/util_define.h b/cpp/src/utils/util_define.h index f7f638e5b..33784e152 100644 --- a/cpp/src/utils/util_define.h +++ b/cpp/src/utils/util_define.h @@ -120,14 +120,14 @@ typedef int mode_t; #endif // __GNUC__ >= 4 /* ======== nullptr ======== */ -#if __cplusplus < 201103L +#if __cplusplus < 201103L && !defined(_MSC_VER) #ifndef nullptr #define nullptr NULL #endif #define OVERRIDE #else #define OVERRIDE override -#endif // __cplusplus < 201103L +#endif // __cplusplus < 201103L && !defined(_MSC_VER) /* ======== cache line ======== */ #ifndef CACHE_LINE_SIZE diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 053e25174..828c6630a 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -41,6 +41,11 @@ raise "All native target runtimes must be checked" unless windows.include?("CheckStaticMSVCRuntime.cmake") raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 +homebrew_merge = jobs.fetch("merge-homebrew").fetch("steps").filter_map { |step| step["run"] }.join("\n") +trust = homebrew_merge.index("brew trust apache/tsfile-dev") +merge = homebrew_merge.index("brew bottle --merge") +raise "Homebrew merge must trust its temporary tap before loading the Formula" unless trust && merge && trust < merge + %w[test-deb test-rpm build-windows].each do |name| steps = jobs.fetch(name).fetch("steps") raise "#{name} must check out the consumer fixture" unless steps.any? { |step| step["uses"].to_s.start_with?("actions/checkout@") } diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py index ed3eb5a10..0ce6d8b42 100644 --- a/packaging/tests/test_native_install.py +++ b/packaging/tests/test_native_install.py @@ -32,10 +32,44 @@ class PublicHeaderCompatibilityTest(unittest.TestCase): - def test_static_assert_uses_the_native_keyword_for_msvc(self): + def test_simd_int64_reduction_normalizes_lane_types(self): + statistic = (ROOT / "cpp/src/common/statistic.h").read_text() + self.assertIn("std::min(simde_mm_cvtsi128_si64(vmin)", statistic) + self.assertIn("std::max(simde_mm_cvtsi128_si64(vmax)", statistic) + source = textwrap.dedent("""\ + #include + #include + + long simde_mm_cvtsi128_si64(int); + long long simde_mm_extract_epi64(int, int); + + std::int64_t reduce_min(int value) { + return std::min(simde_mm_cvtsi128_si64(value), + simde_mm_extract_epi64(value, 1)); + } + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-fsyntax-only", + "-x", + "c++", + "-std=c++11", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + + def test_msvc_old_cplusplus_uses_native_cxx11_features(self): source = textwrap.dedent("""\ #include "utils/util_define.h" class Probe { STATIC_ASSERT(true, MSVC_old_cplusplus); }; + class Base { virtual void reset(); }; + class Derived : public Base { void reset() OVERRIDE; }; """) result = subprocess.run( [ @@ -59,6 +93,46 @@ class Probe { STATIC_ASSERT(true, MSVC_old_cplusplus); }; self.assertEqual(result.returncode, 0, result.stdout[-4000:]) self.assertRegex(result.stdout, r"class Probe \{\s*(?:static_assert|_Static_assert)") self.assertNotIn("static_assertion_MSVC_old_cplusplus", result.stdout) + self.assertIn("void reset() override", result.stdout) + + def test_msvc_old_cplusplus_does_not_define_nullptr(self): + compatibility = (ROOT / "cpp/src/utils/util_define.h").read_text() + section = re.search( + r"/\* ======== nullptr ======== \*/\n(.*?)\n/\* ======== cache line", + compatibility, + re.S, + ) + self.assertIsNotNone(section) + source = section.group(1) + textwrap.dedent("""\ + + #ifdef nullptr + #error nullptr must remain a native MSVC keyword + #endif + class Base { virtual void reset(); }; + class Derived : public Base { void reset() OVERRIDE; }; + int* pointer = nullptr; + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-E", + "-P", + "-nostdinc", + "-x", + "c++", + "-std=c++11", + "-D_MSC_VER=1930", + "-D__cplusplus=199711L", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn("void reset() override", result.stdout) + self.assertRegex(result.stdout, r"int\s*\*\s*pointer\s*=\s*nullptr") class StaticANTLRPackageTest(unittest.TestCase): From 72abe301c86a27be693093a556b6a17bd5282dd9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 17 Sep 2026 15:35:19 +0800 Subject: [PATCH 19/27] fix(packaging): derive package test versions from the checkout The native package version tests hardcoded the fork's 2.5.0 development version, so they failed on any checkout that declares a different cpp/CMakeLists.txt version. Read the version under test instead and keep only the unit cases that intentionally pin an input version. --- .gitignore | 2 ++ .../tests/test_native_package_versions.py | 26 ++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index cf943dae7..a447d8353 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,8 @@ python/tsfile/**/*so* python/data python/venv/* python/tests/__pycache__/* +packaging/**/__pycache__/ +packaging/**/*.py[cod] python/tests/*.tsfile python/tsfile/include diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py index b5a6b80c9..15264486a 100644 --- a/packaging/tests/test_native_package_versions.py +++ b/packaging/tests/test_native_package_versions.py @@ -17,6 +17,7 @@ import importlib.util import json +import re import subprocess import sys import tempfile @@ -27,6 +28,16 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[2] MODULE_PATH = REPOSITORY_ROOT / "packaging/scripts/native_package_versions.py" CPP_CMAKE_FILE = REPOSITORY_ROOT / "cpp/CMakeLists.txt" +CPP_VERSION_PATTERN = re.compile(r"^set\(TsFile_CPP_VERSION\s+(\S+)\)", re.MULTILINE) + + +def repository_cpp_version(): + """Return the C++ version declared by the checkout under test.""" + + match = CPP_VERSION_PATTERN.search(CPP_CMAKE_FILE.read_text(encoding="utf-8")) + if match is None: + raise AssertionError(f"cannot find TsFile_CPP_VERSION in {CPP_CMAKE_FILE}") + return match.group(1) def load_version_module(): @@ -54,7 +65,7 @@ def require_module(self): def test_reads_development_version_from_cpp_cmake(self): module = self.require_module() - self.assertEqual(module.read_cpp_version(CPP_CMAKE_FILE), "2.5.0.dev") + self.assertEqual(module.read_cpp_version(CPP_CMAKE_FILE), repository_cpp_version()) def test_builds_exact_development_package_versions(self): module = self.require_module() @@ -106,7 +117,9 @@ def test_rejects_invalid_build_identity_fields(self): def test_cli_writes_json_and_github_outputs(self): module = self.require_module() - expected = module.build_versions("2.5.0.dev", "20260910", 123, 1, "abcdef123456") + expected = module.build_versions( + repository_cpp_version(), "20260910", 123, 1, "abcdef123456" + ) with tempfile.TemporaryDirectory() as temporary_directory: temporary_path = Path(temporary_directory) @@ -142,6 +155,11 @@ def test_cli_writes_json_and_github_outputs(self): ) def test_cli_separates_unterminated_github_output_without_changing_bytes(self): + module = self.require_module() + base_version = module.build_versions( + repository_cpp_version(), "20260910", 123, 1, "abcdef123456" + )["base_version"] + for existing in ( b"", b"existing=value", @@ -177,7 +195,9 @@ def test_cli_separates_unterminated_github_output_without_changing_bytes(self): separator = b"\n" if existing and not existing.endswith(b"\n") else b"" self.assertTrue( output.read_bytes().startswith( - existing + separator + b"base_version=2.5.0\n" + existing + + separator + + f"base_version={base_version}\n".encode("utf-8") ), output.read_bytes(), ) From 46a7a86ba71fc3b26c3f265c03864ba451bb42db Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 16:01:38 +0800 Subject: [PATCH 20/27] build(python): add -Pwith-python-only to reuse the C++ build - pom.xml: add the with-python-only profile plus the shared check default - python/pom.xml: resolve tsfile.cpp.build, pass it to setup.py via TSFILE_CPP_BUILD, and validate the C++ artifacts at the validate phase - python/check_cpp_build.py: fail fast with an actionable message - wheels.yml: drop the explicit -Denable.lzma2=ON so Python wheels follow the single LZMA2 default owned by cpp/pom.xml --- .github/workflows/wheels.yml | 3 -- pom.xml | 24 +++++++++++++ python/check_cpp_build.py | 70 ++++++++++++++++++++++++++++++++++++ python/pom.xml | 36 +++++++++++++++++++ 4 files changed, 130 insertions(+), 3 deletions(-) create mode 100755 python/check_cpp_build.py diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 8161cda0c..53ac76607 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -97,7 +97,6 @@ jobs: -DskipTests -Dspotless.check.skip=true -Dspotless.apply.skip=true \ -Dbuild.test=OFF \ -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ -Dcmake.args="-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0" otool -l cpp/target/build/lib/libtsfile*.dylib | grep -A2 LC_VERSION_MIN_MACOSX || true @@ -140,7 +139,6 @@ jobs: ./mvnw -Pwith-cpp clean package \ -DskipTests -Dbuild.test=OFF \ -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ -Dspotless.check.skip=true -Dspotless.apply.skip=true test -d cpp/target/build/lib && test -d cpp/target/build/include @@ -208,7 +206,6 @@ jobs: ./mvnw -Pwith-cpp clean package \ -DskipTests -Dbuild.test=OFF \ -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ -Dspotless.check.skip=true -Dspotless.apply.skip=true test -d cpp/target/build/lib test -d cpp/target/build/include diff --git a/pom.xml b/pom.xml index c89304eab..524dedc88 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,14 @@ pom Apache TsFile Project Parent POM + + true 17 17 17 @@ -620,6 +628,22 @@ python + + + with-python-only + + python + + + ${maven.multiModuleProjectDirectory}/cpp/target/build + false + + .java-17-and-above diff --git a/python/check_cpp_build.py b/python/check_cpp_build.py new file mode 100755 index 000000000..b8e6f61a8 --- /dev/null +++ b/python/check_cpp_build.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +"""Validate that the C++ build tree required by the Python module exists. + +The Python module links against the native library produced by the ``cpp`` +module. When the two are built together (``-Pwith-python``) the artifacts are +always present. When the Python module is built on its own +(``-Pwith-python-only``) they must already exist, so this check turns a late and +opaque failure inside ``setup.py`` into an early, actionable one. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def missing_parts(build_root: Path) -> list[Path]: + """Return the required subdirectories of *build_root* that are absent.""" + return [ + part + for part in (build_root / "include", build_root / "lib") + if not part.is_dir() + ] + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + print(f"usage: {Path(argv[0]).name} ", file=sys.stderr) + return 2 + + build_root = Path(argv[1]).expanduser().resolve() + missing = missing_parts(build_root) + if not missing: + return 0 + + print( + f"error: TsFile C++ build not found at {build_root}", + file=sys.stderr, + ) + for part in missing: + print(f" missing directory: {part}", file=sys.stderr) + print( + " Run './mvnw -Pwith-cpp package' first, " + "or set -Dtsfile.cpp.build=.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/python/pom.xml b/python/pom.xml index 8de3d2fbb..ca5777941 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -29,6 +29,13 @@ TsFile: Python 2.6.0 + + ${project.basedir}/../cpp/target/build tsfile ${project.build.directory}/build-wrapper-output @@ -69,7 +76,36 @@ org.codehaus.mojo exec-maven-plugin + + + + ${tsfile.cpp.build} + + + + + check-cpp-build + validate + + exec + + + ${tsfile.cpp.check.skip} + ${python.exe.bin} + + ${project.basedir}/check_cpp_build.py + ${tsfile.cpp.build} + + + python-venv From 89fbeda713ecc8b96357a9fba6561d0cc793750a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 16:09:37 +0800 Subject: [PATCH 21/27] build(cpp): expose CPack generation via -Denable.cpack=ON Let the build tree produced by '-Pwith-cpp package' also generate the CPack configuration, so native packages can be produced from that same tree without reconfiguring or rebuilding it. --- cpp/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/pom.xml b/cpp/pom.xml index 30c70b858..10035aa5a 100644 --- a/cpp/pom.xml +++ b/cpp/pom.xml @@ -52,6 +52,13 @@ OFF ON ON + + OFF ${project.basedir} @@ -105,6 +112,7 @@ + OFF + + OFF + + ${project.basedir} @@ -113,6 +124,8 @@ + + + + native-package-versions + + + tsfile.archive.version + + + + + + com.googlecode.cmake-maven-project + cmake-maven-plugin + + + cmake-generate-test-compile + + + + + + + + + + + + + + with-code-coverage diff --git a/packaging/README.md b/packaging/README.md index 32b8106d4..f8ca30309 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -32,16 +32,20 @@ API; a separate API cleanup will narrow it in a future version. ## Manual artifact workflow `Build native package artifacts` (`.github/workflows/native-packages.yml`) is -the authoritative native packaging workflow. In the fork's GitHub Actions tab, +the authoritative native packaging workflow. In the Apache TsFile repository's GitHub Actions tab, select this workflow, choose **Run workflow**, select the branch containing the commit to build, and dispatch it manually. It runs only on `workflow_dispatch`, with read-only repository permissions; pushes and pull requests do not trigger native packaging. -A successful run produces Ubuntu DEBs, AlmaLinux RPMs, an ARM64 and Intel -Homebrew development bottle with a merged Formula, and a Windows x86_64 SDK/CLI -ZIP. It combines these into `tsfile-native-packages-` with -`manifest.json` and `SHA256SUMS`. Download this final artifact from the workflow +A successful run builds the C++ core through Maven, stages one SDK per +release platform, and then produces Ubuntu DEBs, AlmaLinux RPMs, an ARM64 and +Intel Homebrew development bottle with a merged Formula, a Windows x86_64 +SDK/CLI ZIP, and a Linux Python wheel built from the staged Ubuntu SDK. It also +runs the Go and Python consumers against the SDK rather than rebuilding the C++ +core downstream. The job combines these into +`tsfile-native-packages-` with `manifest.json` and +`SHA256SUMS`. Download this final artifact from the workflow run; both intermediate and final artifacts are retained for 14 days. Publishing is a separate, manual step. This workflow only builds, tests, and @@ -51,39 +55,34 @@ implement RC/final release behavior. ## Portable archive -Build a relocatable binary archive on any host with CMake and CPack: +Build a relocatable SDK and portable archive through the Maven entry point: ```bash -cmake -S cpp -B cpp/build/package \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST=OFF \ - -DBUILD_TOOLS=ON \ - -DTSFILE_ENABLE_CPACK=ON \ - -DTSFILE_DEPENDENCY_SOURCE=AUTO -cmake --build cpp/build/package --parallel -cpack --config cpp/build/package/CPackConfig.cmake -G TGZ +./mvnw -Pwith-cpp package \ + -Dbuild.test=OFF \ + -Dtsfile.dependency.source=AUTO \ + -Denable.cpack=ON +cmake --install cpp/target/build --prefix cpp/build/sdk-root +cpack --config cpp/target/build/CPackConfig.cmake -G TGZ -B cpp/build/packages ``` -The resulting `tsfile--.tar.gz` contains a standard -prefix layout and can be unpacked at `/usr/local`, a user directory, or a -relocated application prefix. +The staged `cpp/build/sdk-root` contains a standard prefix layout and can be +unpacked at `/usr/local`, a user directory, or a relocated application prefix. ## Native Linux packages DEB and RPM packages must be built in the target distribution environment so CPack can run the native dependency scanner (`dpkg-shlibdeps` or -`rpmbuild`). On Debian/Ubuntu use `-G DEB`; on Fedora/RHEL use `-G RPM`: +`rpmbuild`). Build the same Maven-driven tree once, then select the native +generator in the same job: ```bash -cmake -S cpp -B cpp/build/package \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST=OFF \ - -DBUILD_TOOLS=ON \ - -DTSFILE_ENABLE_CPACK=ON \ - -DTSFILE_DEPENDENCY_SOURCE=AUTO -cmake --build cpp/build/package --parallel -cpack --config cpp/build/package/CPackConfig.cmake -G DEB -# or: cpack --config cpp/build/package/CPackConfig.cmake -G RPM +./mvnw -Pwith-cpp package \ + -Dbuild.test=OFF \ + -Dtsfile.dependency.source=AUTO \ + -Denable.cpack=ON +cpack -G DEB --config cpp/target/build/CPackConfig.cmake -B cpp/build/packages +# or: cpack -G RPM --config cpp/target/build/CPackConfig.cmake -B cpp/build/packages ``` `SYSTEM` can be used instead of `AUTO` when the build image provides every @@ -115,7 +114,7 @@ when the first release containing this packaging work is published. The manual native-package workflow also builds the self-hosted development Formula `tsfile-dev`, using `packaging/homebrew/tsfile-dev.rb.in`. This is not a Homebrew/core submission. Each build pins the full workflow commit from -`ColinLeeo/tsfile`, hashes that source archive, and uses the generated immutable +the Apache TsFile source repository, hashes that source archive, and uses the generated immutable Homebrew development version. The Formula retains the CMake install behavior and tests the installed C++ consumer and CLI before bottling. `tsfile-dev` is keg-only because the current SDK intentionally ships its dependency header @@ -203,8 +202,9 @@ python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM installation test, Homebrew bottle merge, and Windows SDK/CLI build all succeed, the workflow assembles `tsfile-native-packages-`. The final -GitHub Actions artifact retains the DEBs, RPMs, merged Formula and bottles, and -Windows ZIP in their package-family layouts for 14 days. It also contains a +GitHub Actions artifact retains the platform SDKs, the Linux Python wheel, the +DEBs, RPMs, merged Formula and bottles, and Windows ZIP in their package-family +layouts for 14 days. It also contains a sorted `SHA256SUMS` and `manifest.json` with the source identity, generated versions, byte sizes, SHA-256 values, and the JFrog repository, immutable target path, and properties required for later manual publication. @@ -212,5 +212,6 @@ path, and properties required for later manual publication. The final job has no publishing credentials and does not upload to JFrog. A maintainer can later use the manifest to upload DEBs to `tsfile-debian` with the recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, and Homebrew -and Windows files to their immutable `tsfile/homebrew/dev/versions/` -and `tsfile/windows/dev/versions/` paths. +SDKs, and Windows files to their immutable +`tsfile/homebrew/dev/versions/`, +`tsfile/windows/dev/versions/`, and `sdk/dev/versions/` paths. diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py index 51cec3769..963915113 100644 --- a/packaging/scripts/assemble_native_packages.py +++ b/packaging/scripts/assemble_native_packages.py @@ -36,6 +36,12 @@ } REQUIRED_VERSIONS = {"archive_version", "homebrew_version"} REQUIRED_SOURCE = {"commit", "repository"} +SDK_PLATFORMS = { + "ubuntu22.04-amd64", + "almalinux9-x86_64", + "windows-msvc-x86_64", +} +PYTHON_PLATFORMS = {"ubuntu22.04-x86_64"} def _validate_metadata(values: dict[str, str], required: set[str], name: str) -> None: @@ -141,6 +147,34 @@ def _artifact_description( "targetPath": f"homebrew/dev/versions/{versions['homebrew_version']}/{target.as_posix()[len('homebrew/'):]}", "properties": {}, } + if len(parts) == 3 and parts[0] == "sdk" and filename.endswith(".tar.gz"): + platform = parts[1] + if platform not in SDK_PLATFORMS: + raise ValueError(f"unsupported SDK platform: {platform}") + target = Path("sdk") / platform / filename + return target, { + "family": "sdk", + "platform": platform, + "targetRepository": "tsfile", + "targetPath": ( + f"sdk/dev/versions/{versions['archive_version']}/{platform}/{filename}" + ), + "properties": {}, + } + if len(parts) == 3 and parts[0] == "python" and filename.endswith(".whl"): + platform = parts[1] + if platform not in PYTHON_PLATFORMS: + raise ValueError(f"unsupported Python wheel platform: {platform}") + target = Path("python/wheels") / platform / filename + return target, { + "family": "python", + "platform": platform, + "targetRepository": "tsfile-python", + "targetPath": ( + f"python/dev/versions/{versions['archive_version']}/{platform}/{filename}" + ), + "properties": {}, + } if len(parts) >= 2 and parts[0] == "windows" and filename.endswith(".zip"): target = Path("windows") / filename return target, { @@ -163,6 +197,27 @@ def _require_complete_families( raise ValueError("missing RPM package input") if not any(family == "windows" for family in families): raise ValueError("missing Windows ZIP input") + sdk_platforms = { + metadata["platform"] + for _, _, metadata in planned + if metadata["family"] == "sdk" + } + missing_sdk_platforms = SDK_PLATFORMS - sdk_platforms + if missing_sdk_platforms: + raise ValueError( + "missing SDK inputs for: " + ", ".join(sorted(missing_sdk_platforms)) + ) + python_platforms = { + metadata["platform"] + for _, _, metadata in planned + if metadata["family"] == "python" + } + missing_python_platforms = PYTHON_PLATFORMS - python_platforms + if missing_python_platforms: + raise ValueError( + "missing Python wheel inputs for: " + + ", ".join(sorted(missing_python_platforms)) + ) homebrew_paths = { target.as_posix() for _, target, metadata in planned diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 828c6630a..506d8a593 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -6,7 +6,7 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -26,22 +26,51 @@ raise "Workflow permissions must remain read-only" unless workflow["permissions"] == { "contents" => "read" } jobs = workflow.fetch("jobs") -bootstrap = jobs.fetch("build-rpm").fetch("steps").find { |step| step["name"] == "Install RPM build prerequisites" } -packages = Shellwords.split(bootstrap.fetch("run").gsub("\\\n", " ")) + +def run_text(job) + job.fetch("steps").filter_map { |step| step["run"] }.join("\n") +end + +%w[build-deb build-rpm build-windows].each do |name| + text = run_text(jobs.fetch(name)) + raise "#{name} must build C++ through Maven" unless text.include?("-Pwith-cpp package") + raise "#{name} must enable CPack through Maven" unless text.include?("-Denable.cpack=ON") + raise "#{name} must use the generated CPack config" unless text.include?("cpp/target/build/CPackConfig.cmake") + raise "#{name} must stage an SDK" unless text.include?("cmake --install cpp/target/build") + raise "#{name} must pass the generated archive version" unless text.include?("-Dtsfile.archive.version") + raise "#{name} must freeze source versions" unless text.include?("-Dtsfile.version.sync.skip=true") +end + +rpm_bootstrap = jobs.fetch("build-rpm").fetch("steps").find { |step| step["name"] == "Install RPM build prerequisites" } +packages = Shellwords.split(rpm_bootstrap.fetch("run").gsub("\\\n", " ")) raise "AlmaLinux 9 curl-minimal conflicts with full curl" if packages.include?("curl") raise "RPM bootstrap must retain curl-minimal" unless packages.include?("curl-minimal") raise "AlmaLinux 9 default repositories do not provide ninja-build" if packages.include?("ninja-build") raise "RPM bootstrap must install make" unless packages.include?("make") +raise "RPM bootstrap must install Java" unless packages.include?("java-17-openjdk-devel") -rpm_build = jobs.fetch("build-rpm").fetch("steps").filter_map { |step| step["run"] }.join("\n") -raise "RPM build must not require Ninja" if rpm_build.include?("-G Ninja") - -windows = jobs.fetch("build-windows").fetch("steps").filter_map { |step| step["run"] }.join("\n") -raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-DTSFILE_MSVC_STATIC_RUNTIME=ON") && windows.include?("-DTSFILE_DEPENDENCY_SOURCE=BUNDLED") -raise "All native target runtimes must be checked" unless windows.include?("CheckStaticMSVCRuntime.cmake") +windows = run_text(jobs.fetch("build-windows")) +raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-Dtsfile.msvc.static.runtime=ON") && windows.include?("-Dtsfile.dependency.source=BUNDLED") +raise "Windows runtime check must be wired through Maven" unless windows.include?("CheckStaticMSVCRuntime.cmake") && windows.include?("-Dtsfile.project.include") raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 -homebrew_merge = jobs.fetch("merge-homebrew").fetch("steps").filter_map { |step| step["run"] }.join("\n") +go_job = jobs.fetch("test-go-linux") +raise "Go test must consume the Ubuntu SDK" unless go_job.fetch("needs") == "build-deb" +go_text = run_text(go_job) +raise "Go test must download the SDK artifact" unless go_job.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-ubuntu22.04-amd64" } +raise "Go test must run go test" unless go_text.include?("go test ./...") + +python_job = jobs.fetch("build-python-linux") +raise "Python wheel must consume the Ubuntu SDK" unless python_job.fetch("needs") == "build-deb" +python_text = run_text(python_job) +raise "Python wheel must use with-python-only" unless python_text.include?("-Pwith-python-only package") +raise "Python wheel must point at the downloaded SDK" unless python_text.include?("-Dtsfile.cpp.build") +raise "Python wheel must freeze source versions" unless python_text.include?("-Dtsfile.version.sync.skip=true") + +homebrew = jobs.fetch("build-homebrew") +raise "Homebrew must use the current repository" unless homebrew.fetch("env").fetch("SOURCE_REPOSITORY") == "${{ github.repository }}" + +homebrew_merge = run_text(jobs.fetch("merge-homebrew")) trust = homebrew_merge.index("brew trust apache/tsfile-dev") merge = homebrew_merge.index("brew bottle --merge") raise "Homebrew merge must trust its temporary tap before loading the Formula" unless trust && merge && trust < merge @@ -53,6 +82,18 @@ raise "#{name} must use the installed CMake consumer" unless runs.include?("-S cpp/cmake/tests/projects/InstalledConsumer") end +assemble = jobs.fetch("assemble") +needs = Array(assemble.fetch("needs")) +%w[test-deb test-rpm test-go-linux build-python-linux merge-homebrew build-windows].each do |name| + raise "assemble must depend on #{name}" unless needs.include?(name) +end +assemble_text = run_text(assemble) +%w[native-sdk-ubuntu22.04-amd64 native-sdk-almalinux9-x86_64 native-sdk-windows-msvc-x86_64 native-python-wheel-ubuntu22.04-x86_64].each do |name| + raise "assemble must download #{name}" unless assemble.fetch("steps").any? { |step| step["with"].to_h["name"] == name } +end +raise "assemble must verify the bundle" unless assemble_text.include?("--verify-bundle") +raise "workflow must not hard-code a fork repository" if File.read(path).include?("ColinLeeo/tsfile") + jobs.each_value do |job| job.fetch("steps").each do |step| next unless step["run"] && step.fetch("shell", "bash") == "bash" @@ -60,4 +101,4 @@ raise "Invalid shell in #{step['name']}: #{output}" unless status.success? end end -puts "PASS: manual/read-only workflow, AlmaLinux bootstrap, Windows CRT checks, shared consumers, and Bash syntax" +puts "PASS: manual/read-only Maven build hierarchy, artifact consumers, workflow contracts, and shell syntax" diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py index 54da32c66..034a6386c 100644 --- a/packaging/tests/test_assemble_native_packages.py +++ b/packaging/tests/test_assemble_native_packages.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import hashlib import importlib.util import json import os @@ -33,7 +34,7 @@ } SOURCE = { "commit": "abcdef1234567890abcdef1234567890abcdef12", - "repository": "ColinLeeo/tsfile", + "repository": "apache/tsfile", } @@ -69,6 +70,10 @@ def write_fixture(self, directory): "homebrew/bottles/tsfile-dev.bottle.tar.gz": b"bottle artifact\n", "homebrew/bottles/tsfile-dev.bottle.json": b"bottle metadata\n", "windows/tsfile-2.5.0-dev-windows-x86_64.zip": b"windows artifact\n", + "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk ubuntu\n", + "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk almalinux\n", + "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk windows\n", + "python/ubuntu22.04-x86_64/tsfile-2.5.0.dev0.20260910.123.1.gabcdef1-cp311-cp311-linux_x86_64.whl": b"python wheel\n", } for relative_path, contents in files.items(): path = directory / relative_path @@ -93,7 +98,11 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): "homebrew/Formula/tsfile-dev.rb", "homebrew/bottles/tsfile-dev.bottle.json", "homebrew/bottles/tsfile-dev.bottle.tar.gz", + "python/wheels/ubuntu22.04-x86_64/tsfile-2.5.0.dev0.20260910.123.1.gabcdef1-cp311-cp311-linux_x86_64.whl", "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "windows/tsfile-2.5.0-dev-windows-x86_64.zip", ] self.assertEqual( @@ -105,95 +114,37 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): ), expected_paths, ) - self.assertEqual( - (output_directory / "SHA256SUMS").read_text(encoding="utf-8"), - "ef5ca1431457b6dec117aff4aafcfd14edec3fe628d3fefe334867e502b8c137 deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb\n" - "b045d33311f553503342e1b061df60161e414503d68dc14af90537f254dfc224 homebrew/Formula/tsfile-dev.rb\n" - "ae98b99fb84c92fb10b00d75e0c51035006067659043b9dc0cb08d9902633967 homebrew/bottles/tsfile-dev.bottle.json\n" - "95f3ddb71b1a0c5c95c5aa0352321075cc465d6a583dff8bf601e888abec4b82 homebrew/bottles/tsfile-dev.bottle.tar.gz\n" - "f557fbfdaf15a34e59adaf4fa487062419dd8fe39eb057f6b581b4469cd3a25f rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm\n" - "f59fc30fd2aaa4a4594eb006fce33041e2702b621abf936af5619b3eabf23c8f windows/tsfile-2.5.0-dev-windows-x86_64.zip\n", - ) self.assertEqual( json.loads((output_directory / "manifest.json").read_text()), manifest ) self.assertEqual(manifest["source"], SOURCE) self.assertEqual(manifest["versions"], VERSIONS) self.assertEqual( - manifest["artifacts"], - [ - { - "family": "deb", - "filename": "tsfile_2.5.0-dev_amd64.deb", - "path": "deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", - "platform": "ubuntu22.04-amd64", - "properties": { - "deb.architecture": ["amd64"], - "deb.component": ["dev"], - "deb.distribution": ["jammy", "noble"], - }, - "sha256": "ef5ca1431457b6dec117aff4aafcfd14edec3fe628d3fefe334867e502b8c137", - "size": 13, - "targetPath": "pool/dev/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", - "targetRepository": "tsfile-debian", - }, - { - "family": "homebrew", - "filename": "tsfile-dev.rb", - "path": "homebrew/Formula/tsfile-dev.rb", - "platform": "homebrew", - "properties": {}, - "sha256": "b045d33311f553503342e1b061df60161e414503d68dc14af90537f254dfc224", - "size": 17, - "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/Formula/tsfile-dev.rb", - "targetRepository": "tsfile", - }, - { - "family": "homebrew", - "filename": "tsfile-dev.bottle.json", - "path": "homebrew/bottles/tsfile-dev.bottle.json", - "platform": "homebrew", - "properties": {}, - "sha256": "ae98b99fb84c92fb10b00d75e0c51035006067659043b9dc0cb08d9902633967", - "size": 16, - "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles/tsfile-dev.bottle.json", - "targetRepository": "tsfile", - }, - { - "family": "homebrew", - "filename": "tsfile-dev.bottle.tar.gz", - "path": "homebrew/bottles/tsfile-dev.bottle.tar.gz", - "platform": "homebrew", - "properties": {}, - "sha256": "95f3ddb71b1a0c5c95c5aa0352321075cc465d6a583dff8bf601e888abec4b82", - "size": 16, - "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles/tsfile-dev.bottle.tar.gz", - "targetRepository": "tsfile", - }, - { - "family": "rpm", - "filename": "tsfile-2.5.0-dev.x86_64.rpm", - "path": "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", - "platform": "almalinux9-x86_64", - "properties": {}, - "sha256": "f557fbfdaf15a34e59adaf4fa487062419dd8fe39eb057f6b581b4469cd3a25f", - "size": 13, - "targetPath": "dev/el9/x86_64/tsfile-2.5.0-dev.x86_64.rpm", - "targetRepository": "tsfile-rpm", - }, - { - "family": "windows", - "filename": "tsfile-2.5.0-dev-windows-x86_64.zip", - "path": "windows/tsfile-2.5.0-dev-windows-x86_64.zip", - "platform": "windows-msvc-x86_64", - "properties": {}, - "sha256": "f59fc30fd2aaa4a4594eb006fce33041e2702b621abf936af5619b3eabf23c8f", - "size": 17, - "targetPath": "windows/dev/versions/2.5.0-dev0.20260910.123.1.gabcdef1/tsfile-2.5.0-dev-windows-x86_64.zip", - "targetRepository": "tsfile", - }, - ], + {artifact["family"] for artifact in manifest["artifacts"]}, + {"deb", "homebrew", "python", "rpm", "sdk", "windows"}, ) + self.assertEqual( + {artifact["platform"] for artifact in manifest["artifacts"]}, + { + "almalinux9-x86_64", + "homebrew", + "ubuntu22.04-amd64", + "ubuntu22.04-x86_64", + "windows-msvc-x86_64", + }, + ) + for artifact in manifest["artifacts"]: + output_path = output_directory / artifact["path"] + input_path = next( + path + for path in input_directory.rglob(artifact["filename"]) + if path.is_file() + ) + self.assertEqual(artifact["size"], input_path.stat().st_size) + self.assertEqual( + artifact["sha256"], + hashlib.sha256(input_path.read_bytes()).hexdigest(), + ) def test_verifies_assembler_checksum_lines_sorted_by_path(self): module = self.require_module() diff --git a/packaging/tests/test_render_homebrew_formula.py b/packaging/tests/test_render_homebrew_formula.py index a60f9a6c0..f3e500369 100644 --- a/packaging/tests/test_render_homebrew_formula.py +++ b/packaging/tests/test_render_homebrew_formula.py @@ -27,7 +27,7 @@ MODULE_PATH = ROOT / "packaging/scripts/render_homebrew_formula.py" TEMPLATE_PATH = ROOT / "packaging/homebrew/tsfile-dev.rb.in" VALUES = { - "SOURCE_REPOSITORY": "ColinLeeo/tsfile", + "SOURCE_REPOSITORY": "apache/tsfile", "GIT_SHA": "abcdef1234567890abcdef1234567890abcdef12", "HOMEBREW_VERSION": "2.5.0.dev0.20260910.123.1.gabcdef1", "SOURCE_SHA256": "0123456789abcdef" * 4, @@ -66,7 +66,7 @@ def test_renders_exact_immutable_source_and_both_bottle_tags(self): self.assertEqual( rendered, """class TsfileDev < Formula - url "https://github.com/ColinLeeo/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz" + url "https://github.com/apache/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz" version "2.5.0.dev0.20260910.123.1.gabcdef1" sha256 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" """ @@ -115,7 +115,7 @@ def test_rejects_empty_or_non_immutable_source_identity(self): ("GIT_SHA", ""), ("GIT_SHA", "abcdef1"), ("GIT_SHA", "develop"), - ("SOURCE_REPOSITORY", 'ColinLeeo/tsfile"'), + ("SOURCE_REPOSITORY", 'apache/tsfile"'), ("HOMEBREW_VERSION", 'bad"\nversion "injected'), ): with self.subTest(key=key, value=value): @@ -162,7 +162,7 @@ def test_cli_renders_real_template_with_optional_bottle_file(self): rendered = output.read_text(encoding="utf-8") self.assertIn("class TsfileDev < Formula", rendered) self.assertIn( - 'url "https://github.com/ColinLeeo/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz"', + 'url "https://github.com/apache/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz"', rendered, ) self.assertIn('version "2.5.0.dev0.20260910.123.1.gabcdef1"', rendered) diff --git a/python/VersionUpdater.groovy b/python/VersionUpdater.groovy index a586fe936..fa2c5c970 100644 --- a/python/VersionUpdater.groovy +++ b/python/VersionUpdater.groovy @@ -19,6 +19,17 @@ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Synchronize the version in setup.py and the one used in the maven pom. + +// Native package jobs derive their identity from native_package_versions.py. +// Keep that identity stable by leaving source versions untouched in those jobs. +def skipVersionSync = project.properties.getProperty("tsfile.version.sync.skip") +if (skipVersionSync == null) { + skipVersionSync = System.getProperty("tsfile.version.sync.skip") +} +if (skipVersionSync != null && skipVersionSync.equalsIgnoreCase("true")) { + println "Skipping source version synchronization for native packaging" + return +} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// def currentMavenVersion = project.version as String From c4405fda163900c48d086def9aae85234f8af667 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 16:28:01 +0800 Subject: [PATCH 23/27] docs(packaging): describe SDK and wheel consumers --- packaging/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packaging/README.md b/packaging/README.md index f8ca30309..a7bf7092c 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -200,8 +200,9 @@ python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v ## Final native package bundle Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM -installation test, Homebrew bottle merge, and Windows SDK/CLI build all succeed, -the workflow assembles `tsfile-native-packages-`. The final +installation test, Go SDK tests, Linux Python wheel smoke test, Homebrew bottle +merge, and Windows SDK/CLI build all succeed, the workflow assembles +`tsfile-native-packages-`. The final GitHub Actions artifact retains the platform SDKs, the Linux Python wheel, the DEBs, RPMs, merged Formula and bottles, and Windows ZIP in their package-family layouts for 14 days. It also contains a @@ -211,7 +212,8 @@ path, and properties required for later manual publication. The final job has no publishing credentials and does not upload to JFrog. A maintainer can later use the manifest to upload DEBs to `tsfile-debian` with the -recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, and Homebrew -SDKs, and Windows files to their immutable -`tsfile/homebrew/dev/versions/`, -`tsfile/windows/dev/versions/`, and `sdk/dev/versions/` paths. +recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, Homebrew +formula/bottles to `tsfile/homebrew/dev/versions/`, SDKs to +`tsfile/sdk/dev/versions/`, Windows ZIPs to +`tsfile/windows/dev/versions/`, and Python wheels to the +`tsfile-python` repository recorded in the manifest. From 1583130e83f79bc8feac86aea3a914bc01b38f92 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 17:11:30 +0800 Subject: [PATCH 24/27] fix(ci): make native SDK consumers path-safe --- .github/workflows/native-packages.yml | 24 ++++++++++++------------ packaging/tests/check_native_workflow.rb | 2 ++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 64e98bd5f..0633c4ffc 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | sdk_name="tsfile-sdk-ubuntu22.04-amd64-$ARCHIVE_VERSION" - sdk_root="sdk/$sdk_name" + sdk_root="$PWD/sdk/$sdk_name" cmake --install cpp/target/build --prefix "$sdk_root" test -f "$sdk_root/bin/tsfile-cli" test -d "$sdk_root/include" @@ -285,7 +285,7 @@ jobs: shell: bash run: | sdk_name="tsfile-sdk-almalinux9-x86_64-$ARCHIVE_VERSION" - sdk_root="sdk/$sdk_name" + sdk_root="$PWD/sdk/$sdk_name" cmake --install cpp/target/build --prefix "$sdk_root" test -f "$sdk_root/bin/tsfile-cli" test -d "$sdk_root/include" @@ -426,16 +426,16 @@ jobs: $PSNativeCommandUseErrorActionPreference = $true $projectInclude = (Resolve-Path cpp/cmake/tests/CheckStaticMSVCRuntime.cmake).Path .\mvnw.cmd -Pwith-cpp package ` - -Dcpp.toolchain=msvc ` - -Dbuild.type=Release ` - -Dbuild.test=OFF ` - -DskipTests ` - -Dspotless.skip=true ` - -Dtsfile.dependency.source=BUNDLED ` - -Dtsfile.msvc.static.runtime=ON ` + "-Dcpp.toolchain=msvc" ` + "-Dbuild.type=Release" ` + "-Dbuild.test=OFF" ` + "-DskipTests" ` + "-Dspotless.skip=true" ` + "-Dtsfile.dependency.source=BUNDLED" ` + "-Dtsfile.msvc.static.runtime=ON" ` "-Dtsfile.project.include=$projectInclude" ` - -Dtsfile.version.sync.skip=true ` - -Denable.cpack=ON ` + "-Dtsfile.version.sync.skip=true" ` + "-Denable.cpack=ON" ` "-Dtsfile.archive.version=$env:ARCHIVE_VERSION" ` "-Dtsfile.debian.package.version=$env:DEBIAN_VERSION" ` "-Dtsfile.rpm.package.version=$env:RPM_VERSION" ` @@ -452,7 +452,7 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true $sdkName = "tsfile-sdk-windows-msvc-x86_64-$env:ARCHIVE_VERSION" - $sdkRoot = "sdk/$sdkName" + $sdkRoot = Join-Path $PWD "sdk/$sdkName" cmake --install cpp/target/build --config Release --prefix $sdkRoot $requiredPaths = @( "$sdkRoot/bin/tsfile-cli.exe" diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 506d8a593..f6e2996fb 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -37,6 +37,7 @@ def run_text(job) raise "#{name} must enable CPack through Maven" unless text.include?("-Denable.cpack=ON") raise "#{name} must use the generated CPack config" unless text.include?("cpp/target/build/CPackConfig.cmake") raise "#{name} must stage an SDK" unless text.include?("cmake --install cpp/target/build") + raise "#{name} must use an absolute SDK prefix" unless text.include?("$PWD/sdk/") || text.include?('Join-Path $PWD "sdk/') raise "#{name} must pass the generated archive version" unless text.include?("-Dtsfile.archive.version") raise "#{name} must freeze source versions" unless text.include?("-Dtsfile.version.sync.skip=true") end @@ -52,6 +53,7 @@ def run_text(job) windows = run_text(jobs.fetch("build-windows")) raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-Dtsfile.msvc.static.runtime=ON") && windows.include?("-Dtsfile.dependency.source=BUNDLED") raise "Windows runtime check must be wired through Maven" unless windows.include?("CheckStaticMSVCRuntime.cmake") && windows.include?("-Dtsfile.project.include") +raise "Windows Maven properties must be quoted for PowerShell" unless windows.include?('"-Dcpp.toolchain=msvc"') && windows.include?('"-Dbuild.type=Release"') raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 go_job = jobs.fetch("test-go-linux") From a46bf8148c08be59183565417d4c621bbcd16de8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 17:30:21 +0800 Subject: [PATCH 25/27] fix(ci): consume platform SDK layouts correctly --- .github/workflows/native-packages.yml | 16 ++++++++++------ packaging/tests/check_native_workflow.rb | 5 +++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 0633c4ffc..05c96f541 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -124,9 +124,11 @@ jobs: cmake --install cpp/target/build --prefix "$sdk_root" test -f "$sdk_root/bin/tsfile-cli" test -d "$sdk_root/include" - test -d "$sdk_root/lib" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" "$sdk_root/bin/tsfile-cli" --version - PKG_CONFIG_PATH="$sdk_root/lib/pkgconfig" pkg-config --cflags --libs libtsfile + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/deb \ -DCMAKE_PREFIX_PATH="$sdk_root" cmake --build consumer/deb --parallel @@ -289,9 +291,11 @@ jobs: cmake --install cpp/target/build --prefix "$sdk_root" test -f "$sdk_root/bin/tsfile-cli" test -d "$sdk_root/include" - test -d "$sdk_root/lib" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" "$sdk_root/bin/tsfile-cli" --version - PKG_CONFIG_PATH="$sdk_root/lib/pkgconfig" pkg-config --cflags --libs libtsfile + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/rpm \ -DCMAKE_PREFIX_PATH="$sdk_root" cmake --build consumer/rpm --parallel @@ -579,7 +583,7 @@ jobs: mkdir -p sdk-download sdk cpp/target/build sdk_tar="$(find sdk-download -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" test -n "$sdk_tar" - sdk_dir="$(tar -tzf "$sdk_tar" | head -n 1 | cut -d/ -f1)" + sdk_dir="$(tar -tzf "$sdk_tar" | sed -n '1s#/.*##p')" tar -xzf "$sdk_tar" -C sdk cp -a "sdk/$sdk_dir/." cpp/target/build/ test -f cpp/target/build/include/tsfile/cwrapper/tsfile_cwrapper.h @@ -624,7 +628,7 @@ jobs: run: | sdk_tar="$(find sdk-download -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" test -n "$sdk_tar" - sdk_dir="$(tar -tzf "$sdk_tar" | head -n 1 | cut -d/ -f1)" + sdk_dir="$(tar -tzf "$sdk_tar" | sed -n '1s#/.*##p')" mkdir -p sdk tar -xzf "$sdk_tar" -C sdk sdk_root="$PWD/sdk/$sdk_dir" diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index f6e2996fb..81dcb1efe 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -38,6 +38,9 @@ def run_text(job) raise "#{name} must use the generated CPack config" unless text.include?("cpp/target/build/CPackConfig.cmake") raise "#{name} must stage an SDK" unless text.include?("cmake --install cpp/target/build") raise "#{name} must use an absolute SDK prefix" unless text.include?("$PWD/sdk/") || text.include?('Join-Path $PWD "sdk/') + if name != "build-windows" + raise "#{name} must locate pkg-config without assuming lib" unless text.include?('dirname "$sdk_pkgconfig"') + end raise "#{name} must pass the generated archive version" unless text.include?("-Dtsfile.archive.version") raise "#{name} must freeze source versions" unless text.include?("-Dtsfile.version.sync.skip=true") end @@ -61,6 +64,7 @@ def run_text(job) go_text = run_text(go_job) raise "Go test must download the SDK artifact" unless go_job.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-ubuntu22.04-amd64" } raise "Go test must run go test" unless go_text.include?("go test ./...") +raise "Go SDK extraction must not fail on SIGPIPE" if go_text.include?("tar -tzf") && go_text.include?("head -n 1") python_job = jobs.fetch("build-python-linux") raise "Python wheel must consume the Ubuntu SDK" unless python_job.fetch("needs") == "build-deb" @@ -68,6 +72,7 @@ def run_text(job) raise "Python wheel must use with-python-only" unless python_text.include?("-Pwith-python-only package") raise "Python wheel must point at the downloaded SDK" unless python_text.include?("-Dtsfile.cpp.build") raise "Python wheel must freeze source versions" unless python_text.include?("-Dtsfile.version.sync.skip=true") +raise "Python SDK extraction must not fail on SIGPIPE" if python_text.include?("tar -tzf") && python_text.include?("head -n 1") homebrew = jobs.fetch("build-homebrew") raise "Homebrew must use the current repository" unless homebrew.fetch("env").fetch("SOURCE_REPOSITORY") == "${{ github.repository }}" From 2fb44640216d8fce81a245f47553d2759712b475 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 17:50:12 +0800 Subject: [PATCH 26/27] fix(python): resolve staged SDK headers --- python/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/setup.py b/python/setup.py index fe19fc91c..83d0b0b8b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -207,6 +207,9 @@ def finalize_options(self): libraries = [] library_dirs = [str(PKG)] include_dirs = [str(PKG), np.get_include(), str(PKG / "include")] +sdk_include = PKG / "include" / "tsfile" +if sdk_include.is_dir(): + include_dirs.append(str(sdk_include)) if sys.platform.startswith("linux"): libraries = ["tsfile"] From dafad98e1a58ee0715c9fdb5f83ec5ff83f72999 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 18:11:02 +0800 Subject: [PATCH 27/27] fix(python): ship versioned shared libraries --- python/setup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index 83d0b0b8b..a6a26fa42 100644 --- a/python/setup.py +++ b/python/setup.py @@ -92,10 +92,9 @@ def _find_lib(root, patterns): if not candidates: raise FileNotFoundError("missing libtsfile.so* in build output") src = candidates[0] - dst = PKG / src.name - shutil.copy2(src, dst) - link_name = PKG / "libtsfile.so" - shutil.copy2(src, link_name) + for candidate in candidates: + shutil.copy2(candidate, PKG / candidate.name) + shutil.copy2(src, PKG / "libtsfile.so") elif sys.platform == "darwin": candidates = sorted(CPP_LIB.rglob("libtsfile.*.dylib")) or list(