From 3775053abb8d07a9361ef55570d2bf4f50cef6e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sat, 23 May 2026 21:13:16 +0000 Subject: [PATCH 01/13] feat(format): draft arrow.range canonical extension type Add a canonical extension type for bounded ranges (mathematical intervals), distinct from Arrow's calendar Interval (duration) type. - Spec: docs/source/format/CanonicalExtensions.rst adds the Range section. Storage is Struct with both bounds nullable (null = +/-infinity, treated as exclusive). A closed parameter (left/right/both/neither, pandas vocabulary) is carried as JSON extension metadata; the subtype is read from storage. Disambiguates from the calendar Interval type per DB convention (INTERVAL = duration, RANGE/PERIOD = bounded set). - C++ reference impl: cpp/src/arrow/extension/range.{h,cc} (RangeType/RangeArray) with serialize/deserialize, storage validation, registration in the global registry, tests, and CMake/meson wiring. --- cpp/src/arrow/CMakeLists.txt | 1 + cpp/src/arrow/extension/CMakeLists.txt | 3 +- cpp/src/arrow/extension/meson.build | 2 + cpp/src/arrow/extension/range.cc | 206 +++++++++++++++ cpp/src/arrow/extension/range.h | 104 ++++++++ cpp/src/arrow/extension/range_test.cc | 290 +++++++++++++++++++++ cpp/src/arrow/extension_type.cc | 2 + cpp/src/arrow/meson.build | 1 + docs/source/format/CanonicalExtensions.rst | 96 +++++++ 9 files changed, 704 insertions(+), 1 deletion(-) create mode 100644 cpp/src/arrow/extension/range.cc create mode 100644 cpp/src/arrow/extension/range.h create mode 100644 cpp/src/arrow/extension/range_test.cc diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index eead221dbdae..cee9e9fe870e 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -1090,6 +1090,7 @@ if(ARROW_JSON) arrow_add_object_library(ARROW_JSON extension/fixed_shape_tensor.cc extension/opaque.cc + extension/range.cc extension/tensor_internal.cc extension/variable_shape_tensor.cc json/options.cc diff --git a/cpp/src/arrow/extension/CMakeLists.txt b/cpp/src/arrow/extension/CMakeLists.txt index ae52bc32a998..966c927bf2ce 100644 --- a/cpp/src/arrow/extension/CMakeLists.txt +++ b/cpp/src/arrow/extension/CMakeLists.txt @@ -18,7 +18,8 @@ set(CANONICAL_EXTENSION_TESTS bool8_test.cc json_test.cc uuid_test.cc) if(ARROW_JSON) - list(APPEND CANONICAL_EXTENSION_TESTS tensor_extension_array_test.cc opaque_test.cc) + list(APPEND CANONICAL_EXTENSION_TESTS tensor_extension_array_test.cc opaque_test.cc + range_test.cc) endif() add_arrow_test(test diff --git a/cpp/src/arrow/extension/meson.build b/cpp/src/arrow/extension/meson.build index 84dafe4bbe32..8be6b1321a1c 100644 --- a/cpp/src/arrow/extension/meson.build +++ b/cpp/src/arrow/extension/meson.build @@ -21,6 +21,7 @@ if needs_json canonical_extension_tests += [ 'tensor_extension_array_test.cc', 'opaque_test.cc', + 'range_test.cc', ] endif @@ -38,6 +39,7 @@ install_headers( 'json.h', 'opaque.h', 'parquet_variant.h', + 'range.h', 'uuid.h', 'variable_shape_tensor.h', ], diff --git a/cpp/src/arrow/extension/range.cc b/cpp/src/arrow/extension/range.cc new file mode 100644 index 000000000000..2d5d35b2f438 --- /dev/null +++ b/cpp/src/arrow/extension/range.cc @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/extension/range.h" + +#include +#include + +#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep +#include "arrow/util/logging_internal.h" + +#include +#include +#include + +namespace arrow::extension { + +namespace { + +/// Map RangeClosed -> the JSON string value used in serialization. +std::string_view ClosedToString(RangeClosed closed) { + switch (closed) { + case RangeClosed::Left: + return "left"; + case RangeClosed::Right: + return "right"; + case RangeClosed::Both: + return "both"; + case RangeClosed::Neither: + return "neither"; + } + // unreachable + return "right"; +} + +/// Parse the JSON "closed" string into a RangeClosed enum. +/// Returns an error if the string is not one of the four valid values. +Result ClosedFromString(std::string_view s) { + if (s == "left") return RangeClosed::Left; + if (s == "right") return RangeClosed::Right; + if (s == "both") return RangeClosed::Both; + if (s == "neither") return RangeClosed::Neither; + return Status::Invalid( + "Invalid value for RangeType \"closed\" parameter: \"", s, + "\". Expected one of: \"left\", \"right\", \"both\", \"neither\"."); +} + +/// Build the storage Struct type for a given value subtype. +std::shared_ptr MakeStorageType(const std::shared_ptr& value_type) { + // Both "lower" and "upper" are nullable (null = infinite bound). + return struct_({field("lower", value_type, /*nullable=*/true), + field("upper", value_type, /*nullable=*/true)}); +} + +} // namespace + +// --------------------------------------------------------------------------- +// RangeType + +std::shared_ptr RangeType::value_type() const { + // storage_type() is a struct with two fields; both share the same type. + return internal::checked_cast(*storage_type()).field(0)->type(); +} + +std::string RangeType::ToString(bool show_metadata) const { + std::stringstream ss; + ss << "extension<" << extension_name() + << "[value_type=" << value_type()->ToString(show_metadata) + << ", closed=" << ClosedToString(closed_) << "]>"; + return ss.str(); +} + +bool RangeType::ExtensionEquals(const ExtensionType& other) const { + if (extension_name() != other.extension_name()) { + return false; + } + const auto& other_range = internal::checked_cast(other); + return storage_type()->Equals(*other_range.storage_type()) && + closed_ == other_range.closed_; +} + +std::string RangeType::Serialize() const { + rapidjson::Document document; + document.SetObject(); + rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); + + auto closed_str = ClosedToString(closed_); + rapidjson::Value closed_value(closed_str.data(), + static_cast(closed_str.size()), + allocator); + document.AddMember(rapidjson::Value("closed", allocator), closed_value, allocator); + + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + document.Accept(writer); + return buffer.GetString(); +} + +Result> RangeType::Deserialize( + std::shared_ptr storage_type, const std::string& serialized_data) const { + // Validate storage type structure. + if (storage_type->id() != Type::STRUCT) { + return Status::Invalid("RangeType storage type must be a Struct, got ", + storage_type->ToString()); + } + const auto& struct_type = internal::checked_cast(*storage_type); + if (struct_type.num_fields() != 2) { + return Status::Invalid( + "RangeType storage Struct must have exactly 2 fields, got ", + struct_type.num_fields()); + } + const auto& lower_field = struct_type.field(0); + const auto& upper_field = struct_type.field(1); + if (lower_field->name() != "lower") { + return Status::Invalid( + "RangeType storage Struct field 0 must be named \"lower\", got \"", + lower_field->name(), "\""); + } + if (upper_field->name() != "upper") { + return Status::Invalid( + "RangeType storage Struct field 1 must be named \"upper\", got \"", + upper_field->name(), "\""); + } + if (!lower_field->nullable()) { + return Status::Invalid("RangeType storage Struct field \"lower\" must be nullable"); + } + if (!upper_field->nullable()) { + return Status::Invalid("RangeType storage Struct field \"upper\" must be nullable"); + } + if (!lower_field->type()->Equals(*upper_field->type())) { + return Status::Invalid( + "RangeType storage Struct fields \"lower\" and \"upper\" must have the same " + "type, got \"", + lower_field->type()->ToString(), "\" and \"", upper_field->type()->ToString(), + "\""); + } + + // Parse "closed" parameter from JSON metadata. + // Empty metadata defaults to {"closed": "right"}. + RangeClosed closed = RangeClosed::Right; + if (!serialized_data.empty()) { + rapidjson::Document document; + const auto& parsed = + document.Parse(serialized_data.data(), serialized_data.length()); + if (parsed.HasParseError()) { + return Status::Invalid("Invalid serialized JSON data for RangeType: ", + rapidjson::GetParseError_En(parsed.GetParseError()), ": ", + serialized_data); + } + if (!document.IsObject()) { + return Status::Invalid( + "Invalid serialized JSON data for RangeType: not an object"); + } + if (document.HasMember("closed")) { + const auto& closed_val = document["closed"]; + if (!closed_val.IsString()) { + return Status::Invalid( + "Invalid serialized JSON data for RangeType: \"closed\" is not a string"); + } + ARROW_ASSIGN_OR_RAISE( + closed, ClosedFromString(std::string_view(closed_val.GetString(), + closed_val.GetStringLength()))); + } + } + + return std::make_shared(std::move(storage_type), closed); +} + +std::shared_ptr RangeType::MakeArray(std::shared_ptr data) const { + DCHECK_EQ(data->type->id(), Type::EXTENSION); + DCHECK_EQ("arrow.range", + internal::checked_cast(*data->type).extension_name()); + return std::make_shared(data); +} + +Result> RangeType::Make( + std::shared_ptr value_type, RangeClosed closed) { + auto storage = MakeStorageType(value_type); + return std::make_shared(std::move(storage), closed); +} + +// --------------------------------------------------------------------------- +// Free factory function + +std::shared_ptr range(std::shared_ptr value_type, + RangeClosed closed) { + auto result = RangeType::Make(std::move(value_type), closed); + ARROW_CHECK_OK(result.status()); + return std::move(result).ValueOrDie(); +} + +} // namespace arrow::extension diff --git a/cpp/src/arrow/extension/range.h b/cpp/src/arrow/extension/range.h new file mode 100644 index 000000000000..b87f847c4b0c --- /dev/null +++ b/cpp/src/arrow/extension/range.h @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "arrow/extension_type.h" +#include "arrow/type.h" + +namespace arrow::extension { + +/// \brief Which bound(s) of an arrow.range interval are inclusive. +/// +/// Null (infinite) bounds are always exclusive regardless of this value. +enum class RangeClosed { + /// Lower bound is inclusive, upper bound is exclusive: [lower, upper) + Left, + /// Lower bound is exclusive, upper bound is inclusive: (lower, upper] + Right, + /// Both bounds are inclusive: [lower, upper] + Both, + /// Both bounds are exclusive: (lower, upper) + Neither, +}; + +/// \brief RangeType represents a bounded set (mathematical interval) over an +/// orderable Arrow type T. +/// +/// Storage is a Struct with exactly two nullable fields: +/// - "lower": T NULLABLE (null = unbounded below, i.e. -infinity) +/// - "upper": T NULLABLE (null = unbounded above, i.e. +infinity) +/// +/// The outer struct's validity bit marks a null/absent range. +/// +/// The "closed" parameter controls which finite bounds are inclusive. +/// Null (infinite) bounds are always treated as exclusive. +class ARROW_EXPORT RangeType : public ExtensionType { + public: + /// \brief Construct a RangeType. + /// + /// \param[in] storage_type A two-field Struct type with nullable fields + /// "lower" and "upper" of the same orderable Arrow type T. + /// \param[in] closed Which bound(s) are inclusive. + explicit RangeType(std::shared_ptr storage_type, RangeClosed closed) + : ExtensionType(std::move(storage_type)), closed_(closed) {} + + std::string extension_name() const override { return "arrow.range"; } + std::string ToString(bool show_metadata = false) const override; + bool ExtensionEquals(const ExtensionType& other) const override; + std::string Serialize() const override; + Result> Deserialize( + std::shared_ptr storage_type, + const std::string& serialized_data) const override; + + /// \brief Create a RangeArray from ArrayData. + std::shared_ptr MakeArray(std::shared_ptr data) const override; + + /// \brief Factory function. + /// + /// Constructs the required two-field struct storage type internally. + /// \param[in] value_type The orderable Arrow subtype T for lower and upper. + /// \param[in] closed Which bound(s) are inclusive. + static Result> Make(std::shared_ptr value_type, + RangeClosed closed = RangeClosed::Right); + + /// \brief Return the bound-inclusivity parameter. + RangeClosed closed() const { return closed_; } + + /// \brief Return the Arrow subtype T (the type of "lower" and "upper" fields). + std::shared_ptr value_type() const; + + private: + RangeClosed closed_; +}; + +/// \brief Array class for arrow.range extension arrays. +class ARROW_EXPORT RangeArray : public ExtensionArray { + public: + using ExtensionArray::ExtensionArray; +}; + +/// \brief Create a RangeType with the given value subtype and closed parameter. +/// +/// This is a convenience wrapper around RangeType::Make that aborts on error. +/// For recoverable error handling prefer RangeType::Make. +ARROW_EXPORT std::shared_ptr range(std::shared_ptr value_type, + RangeClosed closed = RangeClosed::Right); + +} // namespace arrow::extension diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc new file mode 100644 index 000000000000..eb9c7abe0074 --- /dev/null +++ b/cpp/src/arrow/extension/range_test.cc @@ -0,0 +1,290 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "arrow/extension/range.h" +#include "arrow/extension_type.h" +#include "arrow/io/memory.h" +#include "arrow/ipc/reader.h" +#include "arrow/ipc/writer.h" +#include "arrow/record_batch.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/type.h" +#include "arrow/util/checked_cast.h" + +namespace arrow { + +using internal::checked_pointer_cast; + +// --------------------------------------------------------------------------- +// Helpers + +static std::shared_ptr RangeInt32Right() { + return checked_pointer_cast( + extension::range(int32(), extension::RangeClosed::Right)); +} + +static std::shared_ptr RangeInt32Both() { + return checked_pointer_cast( + extension::range(int32(), extension::RangeClosed::Both)); +} + +static std::shared_ptr RangeInt64Left() { + return checked_pointer_cast( + extension::range(int64(), extension::RangeClosed::Left)); +} + +// --------------------------------------------------------------------------- +// Basics + +TEST(RangeType, Basics) { + auto type = RangeInt32Right(); + ASSERT_EQ("arrow.range", type->extension_name()); + ASSERT_EQ(*int32(), *type->value_type()); + ASSERT_EQ(extension::RangeClosed::Right, type->closed()); + ASSERT_EQ(*type, *type); + ASSERT_NE(*arrow::null(), *type); + ASSERT_THAT(type->Serialize(), ::testing::Not(::testing::IsEmpty())); + ASSERT_EQ(R"({"closed":"right"})", type->Serialize()); + ASSERT_EQ("extension", + type->ToString(false)); +} + +TEST(RangeType, AllClosedValues) { + using C = extension::RangeClosed; + auto left = checked_pointer_cast( + extension::range(int32(), C::Left)); + auto right = checked_pointer_cast( + extension::range(int32(), C::Right)); + auto both = checked_pointer_cast( + extension::range(int32(), C::Both)); + auto neither = checked_pointer_cast( + extension::range(int32(), C::Neither)); + + ASSERT_EQ(R"({"closed":"left"})", left->Serialize()); + ASSERT_EQ(R"({"closed":"right"})", right->Serialize()); + ASSERT_EQ(R"({"closed":"both"})", both->Serialize()); + ASSERT_EQ(R"({"closed":"neither"})", neither->Serialize()); +} + +// --------------------------------------------------------------------------- +// Equals + +TEST(RangeType, Equals) { + auto type_i32_right = RangeInt32Right(); + auto type_i32_both = RangeInt32Both(); + auto type_i64_left = RangeInt64Left(); + auto type_i32_right2 = RangeInt32Right(); + + // Same object. + ASSERT_EQ(*type_i32_right, *type_i32_right); + + // Different instances but same parameters. + ASSERT_EQ(*type_i32_right, *type_i32_right2); + + // Different closed value. + ASSERT_NE(*type_i32_right, *type_i32_both); + + // Different value_type. + ASSERT_NE(*type_i32_right, *type_i64_left); + + // Not equal to a non-range type. + ASSERT_NE(*type_i32_right, *arrow::null()); + ASSERT_NE(*type_i32_right, *arrow::int32()); +} + +// --------------------------------------------------------------------------- +// CreateFromArray + +TEST(RangeType, CreateFromArray) { + auto type = RangeInt32Right(); + // Build a StructArray that matches the storage type. + auto storage_type = type->storage_type(); + auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); + auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); + ASSERT_OK_AND_ASSIGN(auto storage, StructArray::Make({lower, upper}, + {field("lower", int32(), true), + field("upper", int32(), true)})); + auto array = ExtensionType::WrapArray(type, storage); + ASSERT_EQ(3, array->length()); + ASSERT_EQ(0, array->null_count()); +} + +// --------------------------------------------------------------------------- +// Deserialize - valid cases + +void CheckDeserialize(const std::string& serialized, + const std::shared_ptr& expected) { + auto type = checked_pointer_cast(expected); + ASSERT_OK_AND_ASSIGN(auto deserialized, + type->Deserialize(type->storage_type(), serialized)); + ASSERT_EQ(*expected, *deserialized); +} + +TEST(RangeType, Deserialize) { + // Normal JSON + ASSERT_NO_FATAL_FAILURE( + CheckDeserialize(R"({"closed": "right"})", + extension::range(int32(), extension::RangeClosed::Right))); + ASSERT_NO_FATAL_FAILURE( + CheckDeserialize(R"({"closed": "left"})", + extension::range(int32(), extension::RangeClosed::Left))); + ASSERT_NO_FATAL_FAILURE( + CheckDeserialize(R"({"closed": "both"})", + extension::range(int32(), extension::RangeClosed::Both))); + ASSERT_NO_FATAL_FAILURE( + CheckDeserialize(R"({"closed": "neither"})", + extension::range(int32(), extension::RangeClosed::Neither))); + + // Extra fields are tolerated (forward-compatibility). + ASSERT_NO_FATAL_FAILURE( + CheckDeserialize(R"({"closed": "right", "extra": 42})", + extension::range(int32(), extension::RangeClosed::Right))); + + // Empty metadata defaults to "right". + ASSERT_NO_FATAL_FAILURE( + CheckDeserialize("", extension::range(int32(), extension::RangeClosed::Right))); + + // Empty JSON object (no "closed" key) also defaults to "right". + ASSERT_NO_FATAL_FAILURE( + CheckDeserialize("{}", extension::range(int32(), extension::RangeClosed::Right))); +} + +// --------------------------------------------------------------------------- +// Deserialize - invalid cases + +TEST(RangeType, DeserializeInvalidMetadata) { + auto type = RangeInt32Right(); + + // Empty string is valid (defaults to "right"); truly malformed JSON fails. + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("Missing a name for object member"), + type->Deserialize(type->storage_type(), "{")); + + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("not an object"), + type->Deserialize(type->storage_type(), "[]")); + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("\"closed\" is not a string"), + type->Deserialize(type->storage_type(), R"({"closed": 42})")); + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("Invalid value for RangeType"), + type->Deserialize(type->storage_type(), R"({"closed": "unknown"})")); +} + +TEST(RangeType, DeserializeInvalidStorage) { + auto type = RangeInt32Right(); + auto wrong_storage_not_struct = int32(); + + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("must be a Struct"), + type->Deserialize(wrong_storage_not_struct, R"({"closed":"right"})")); + + // Wrong number of fields. + auto one_field = struct_({field("lower", int32(), true)}); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("exactly 2 fields"), + type->Deserialize(one_field, R"({"closed":"right"})")); + + // Wrong field name for field 0. + auto bad_lower_name = + struct_({field("start", int32(), true), field("upper", int32(), true)}); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("named \"lower\""), + type->Deserialize(bad_lower_name, R"({"closed":"right"})")); + + // Wrong field name for field 1. + auto bad_upper_name = + struct_({field("lower", int32(), true), field("end", int32(), true)}); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("named \"upper\""), + type->Deserialize(bad_upper_name, R"({"closed":"right"})")); + + // Fields have different types. + auto mismatched_types = + struct_({field("lower", int32(), true), field("upper", int64(), true)}); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("same type"), + type->Deserialize(mismatched_types, R"({"closed":"right"})")); + + // Non-nullable lower field. + auto lower_not_nullable = + struct_({field("lower", int32(), /*nullable=*/false), field("upper", int32(), true)}); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("\"lower\" must be nullable"), + type->Deserialize(lower_not_nullable, R"({"closed":"right"})")); + + // Non-nullable upper field. + auto upper_not_nullable = + struct_({field("lower", int32(), true), field("upper", int32(), /*nullable=*/false)}); + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("\"upper\" must be nullable"), + type->Deserialize(upper_not_nullable, R"({"closed":"right"})")); +} + +// --------------------------------------------------------------------------- +// Metadata (Serialize/Deserialize) round-trip + +TEST(RangeType, MetadataRoundTrip) { + using C = extension::RangeClosed; + for (const auto& type : + {extension::range(int32(), C::Left), extension::range(int32(), C::Right), + extension::range(int32(), C::Both), extension::range(int32(), C::Neither), + extension::range(int64(), C::Right), extension::range(date32(), C::Both)}) { + auto rt = checked_pointer_cast(type); + std::string serialized = rt->Serialize(); + ASSERT_OK_AND_ASSIGN(auto deserialized, + rt->Deserialize(rt->storage_type(), serialized)); + ASSERT_EQ(*type, *deserialized) << "Round-trip failed for: " << type->ToString(); + } +} + +// --------------------------------------------------------------------------- +// IPC (BatchRoundTrip) -- registration round-trip + +TEST(RangeType, BatchRoundTrip) { + auto type = RangeInt32Right(); + auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); + auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); + ASSERT_OK_AND_ASSIGN(auto storage, StructArray::Make({lower, upper}, + {field("lower", int32(), true), + field("upper", int32(), true)})); + auto array = ExtensionType::WrapArray(type, storage); + auto batch = + RecordBatch::Make(schema({field("rng", type)}), array->length(), {array}); + + std::shared_ptr written; + { + ASSERT_OK_AND_ASSIGN(auto out_stream, io::BufferOutputStream::Create()); + ASSERT_OK(ipc::WriteRecordBatchStream({batch}, ipc::IpcWriteOptions::Defaults(), + out_stream.get())); + ASSERT_OK_AND_ASSIGN(auto complete_ipc_stream, out_stream->Finish()); + + io::BufferReader reader(complete_ipc_stream); + std::shared_ptr batch_reader; + ASSERT_OK_AND_ASSIGN(batch_reader, ipc::RecordBatchStreamReader::Open(&reader)); + ASSERT_OK(batch_reader->ReadNext(&written)); + } + + ASSERT_EQ(*batch->schema(), *written->schema()); + ASSERT_BATCHES_EQUAL(*batch, *written); +} + +} // namespace arrow diff --git a/cpp/src/arrow/extension_type.cc b/cpp/src/arrow/extension_type.cc index ce88c9517411..45cf5ed1e16f 100644 --- a/cpp/src/arrow/extension_type.cc +++ b/cpp/src/arrow/extension_type.cc @@ -31,6 +31,7 @@ #ifdef ARROW_JSON # include "arrow/extension/fixed_shape_tensor.h" # include "arrow/extension/opaque.h" +# include "arrow/extension/range.h" # include "arrow/extension/variable_shape_tensor.h" #endif #include "arrow/extension/json.h" @@ -156,6 +157,7 @@ static void CreateGlobalRegistry() { #ifdef ARROW_JSON ext_types.push_back(extension::fixed_shape_tensor(int64(), {})); ext_types.push_back(extension::opaque(null(), "", "")); + ext_types.push_back(extension::range(int32(), extension::RangeClosed::Right)); ext_types.push_back(extension::variable_shape_tensor(int64(), 0)); #endif diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build index fea26ef4e452..3aaacce0f859 100644 --- a/cpp/src/arrow/meson.build +++ b/cpp/src/arrow/meson.build @@ -529,6 +529,7 @@ if needs_json 'sources': [ 'extension/fixed_shape_tensor.cc', 'extension/opaque.cc', + 'extension/range.cc', 'extension/tensor_internal.cc', 'extension/variable_shape_tensor.cc', 'json/chunked_builder.cc', diff --git a/docs/source/format/CanonicalExtensions.rst b/docs/source/format/CanonicalExtensions.rst index f1faa4a9e741..3e08ca87458b 100644 --- a/docs/source/format/CanonicalExtensions.rst +++ b/docs/source/format/CanonicalExtensions.rst @@ -573,6 +573,102 @@ This extension type is intended to be compatible with ANSI SQL's ``TIMESTAMP WIT It is also *permissible* for the ``offset_minutes`` field to be dictionary-encoded or run-end-encoded. +.. _range_extension: + +Range +===== + +Range represents a bounded set (mathematical interval) defined by a lower and +an upper bound over an orderable Arrow type T. It is the Arrow equivalent of +PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. + +.. note:: + + **Disambiguation from Arrow's calendar** ``Interval`` **type.** + Arrow already has an ``Interval`` type (``INTERVAL_MONTHS``, + ``INTERVAL_DAY_TIME``, ``INTERVAL_MONTH_DAY_NANO``) that represents a + *duration* -- a signed difference between two points in time. The + ``arrow.range`` extension type is an entirely different concept: it + represents a *bounded set* with explicit lower and upper endpoints, + analogous to a closed or open interval in mathematics. The naming + follows database convention: SQL uses ``INTERVAL`` for durations and + ``RANGE`` (or ``PERIOD``) for bounded sets. + +* Extension name: ``arrow.range``. + +* The storage type of the extension is a ``Struct`` with exactly **two fields, + in order**: + + * ``lower``: the lower bound, type **T**, **nullable**. + A null value means the range is unbounded below (negative infinity). + * ``upper``: the upper bound, type **T**, **nullable**. + A null value means the range is unbounded above (positive infinity). + + **T** (the *subtype* or *value type*) may be any orderable Arrow type: + integer, floating-point, decimal, date, time, or timestamp types. Both + fields share the same type T. The subtype is read directly from the + storage struct and is **not** duplicated in the extension metadata. + + The outer struct's validity bit marks a null/absent range (a missing range, + distinct from an empty range). + +.. note:: + + Both ``lower`` and ``upper`` struct fields **must** be nullable. A null + bound represents an infinite endpoint and is **always treated as + exclusive**, regardless of the value of the ``closed`` parameter. You + cannot include positive or negative infinity in a closed bound. + +* Extension type parameters: + + * **closed** = which finite bound(s) are inclusive. Allowed values + (following pandas interval vocabulary): + + * ``"left"`` -- lower bound inclusive, upper bound exclusive: ``[lower, upper)`` + * ``"right"`` -- lower bound exclusive, upper bound inclusive: ``(lower, upper]`` + * ``"both"`` -- both bounds inclusive: ``[lower, upper]`` + * ``"neither"`` -- both bounds exclusive: ``(lower, upper)`` + +* Description of the serialization: + + The extension metadata **must** be either an empty string or a valid JSON + object. The JSON object may contain one key: + + * ``"closed"`` (string, optional): one of ``"left"``, ``"right"``, + ``"both"``, or ``"neither"``. When absent (including when the metadata is + an empty string), it defaults to ``"right"``. + + Additional keys in the JSON object should be ignored to allow + forward-compatible extensions. + + Examples: + + - ``{"closed": "right"}`` -- half-open interval, right-closed (default) + - ``{"closed": "left"}`` -- half-open interval, left-closed + - ``{"closed": "both"}`` -- closed interval + - ``{"closed": "neither"}``-- open interval + - ``""`` -- equivalent to ``{"closed": "right"}`` + +* Semantics: + + * A range value ``[lower, upper]`` with ``closed="both"`` contains every + value x such that ``lower <= x <= upper``. + * A range value ``[lower, upper]`` with ``closed="neither"`` is *empty* + when ``lower == upper`` (a degenerate open interval), and contains values + x such that ``lower < x < upper`` otherwise. + * Implementations should document behavior when ``lower > upper``; the + recommended interpretation is that such a range is *empty*. + * A null outer struct value represents a missing (absent) range, not an + empty range. + * A null ``lower`` field means the range has no lower bound (extends to + negative infinity); the lower bound is always exclusive in this case. + * A null ``upper`` field means the range has no upper bound (extends to + positive infinity); the upper bound is always exclusive in this case. + * A range where both ``lower`` and ``upper`` are null represents the + universal range ``(-inf, +inf)``. + +.. _range types: https://www.postgresql.org/docs/current/rangetypes.html + Community Extension Types ========================= From 84b38be8b0c54247a378557d403ca477eebd8863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sat, 23 May 2026 21:18:14 +0000 Subject: [PATCH 02/13] fix(format): require explicit closed in arrow.range metadata The closedness is no longer defaulted on the wire: empty metadata or a JSON object without a "closed" key is now rejected by Deserialize, so a serialized arrow.range is always unambiguous. The C++ convenience default argument for constructing a RangeType in code is left-closed ([lower, upper)), matching the PostgreSQL/Rust/Python range convention. Spec and tests updated. --- cpp/src/arrow/extension/range.cc | 55 ++++++++++++---------- cpp/src/arrow/extension/range.h | 4 +- cpp/src/arrow/extension/range_test.cc | 26 ++++++---- cpp/src/arrow/extension_type.cc | 2 +- docs/source/format/CanonicalExtensions.rst | 16 ++++--- 5 files changed, 59 insertions(+), 44 deletions(-) diff --git a/cpp/src/arrow/extension/range.cc b/cpp/src/arrow/extension/range.cc index 2d5d35b2f438..97733fe04010 100644 --- a/cpp/src/arrow/extension/range.cc +++ b/cpp/src/arrow/extension/range.cc @@ -149,33 +149,36 @@ Result> RangeType::Deserialize( "\""); } - // Parse "closed" parameter from JSON metadata. - // Empty metadata defaults to {"closed": "right"}. - RangeClosed closed = RangeClosed::Right; - if (!serialized_data.empty()) { - rapidjson::Document document; - const auto& parsed = - document.Parse(serialized_data.data(), serialized_data.length()); - if (parsed.HasParseError()) { - return Status::Invalid("Invalid serialized JSON data for RangeType: ", - rapidjson::GetParseError_En(parsed.GetParseError()), ": ", - serialized_data); - } - if (!document.IsObject()) { - return Status::Invalid( - "Invalid serialized JSON data for RangeType: not an object"); - } - if (document.HasMember("closed")) { - const auto& closed_val = document["closed"]; - if (!closed_val.IsString()) { - return Status::Invalid( - "Invalid serialized JSON data for RangeType: \"closed\" is not a string"); - } - ARROW_ASSIGN_OR_RAISE( - closed, ClosedFromString(std::string_view(closed_val.GetString(), - closed_val.GetStringLength()))); - } + // Parse the required "closed" parameter from JSON metadata. The closedness + // is not defaulted on the wire: empty metadata or a missing key is invalid. + if (serialized_data.empty()) { + return Status::Invalid( + "RangeType metadata must be a JSON object with a required \"closed\" key, " + "got an empty string"); + } + rapidjson::Document document; + const auto& parsed = document.Parse(serialized_data.data(), serialized_data.length()); + if (parsed.HasParseError()) { + return Status::Invalid("Invalid serialized JSON data for RangeType: ", + rapidjson::GetParseError_En(parsed.GetParseError()), ": ", + serialized_data); + } + if (!document.IsObject()) { + return Status::Invalid("Invalid serialized JSON data for RangeType: not an object"); + } + if (!document.HasMember("closed")) { + return Status::Invalid( + "RangeType metadata is missing the required \"closed\" key: ", serialized_data); + } + const auto& closed_val = document["closed"]; + if (!closed_val.IsString()) { + return Status::Invalid( + "Invalid serialized JSON data for RangeType: \"closed\" is not a string"); } + ARROW_ASSIGN_OR_RAISE( + RangeClosed closed, + ClosedFromString( + std::string_view(closed_val.GetString(), closed_val.GetStringLength()))); return std::make_shared(std::move(storage_type), closed); } diff --git a/cpp/src/arrow/extension/range.h b/cpp/src/arrow/extension/range.h index b87f847c4b0c..3af60c1937ab 100644 --- a/cpp/src/arrow/extension/range.h +++ b/cpp/src/arrow/extension/range.h @@ -76,7 +76,7 @@ class ARROW_EXPORT RangeType : public ExtensionType { /// \param[in] value_type The orderable Arrow subtype T for lower and upper. /// \param[in] closed Which bound(s) are inclusive. static Result> Make(std::shared_ptr value_type, - RangeClosed closed = RangeClosed::Right); + RangeClosed closed = RangeClosed::Left); /// \brief Return the bound-inclusivity parameter. RangeClosed closed() const { return closed_; } @@ -99,6 +99,6 @@ class ARROW_EXPORT RangeArray : public ExtensionArray { /// This is a convenience wrapper around RangeType::Make that aborts on error. /// For recoverable error handling prefer RangeType::Make. ARROW_EXPORT std::shared_ptr range(std::shared_ptr value_type, - RangeClosed closed = RangeClosed::Right); + RangeClosed closed = RangeClosed::Left); } // namespace arrow::extension diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc index eb9c7abe0074..2988439986c4 100644 --- a/cpp/src/arrow/extension/range_test.cc +++ b/cpp/src/arrow/extension/range_test.cc @@ -156,14 +156,14 @@ TEST(RangeType, Deserialize) { ASSERT_NO_FATAL_FAILURE( CheckDeserialize(R"({"closed": "right", "extra": 42})", extension::range(int32(), extension::RangeClosed::Right))); +} - // Empty metadata defaults to "right". - ASSERT_NO_FATAL_FAILURE( - CheckDeserialize("", extension::range(int32(), extension::RangeClosed::Right))); - - // Empty JSON object (no "closed" key) also defaults to "right". - ASSERT_NO_FATAL_FAILURE( - CheckDeserialize("{}", extension::range(int32(), extension::RangeClosed::Right))); +TEST(RangeType, DefaultClosedIsLeft) { + // The C++ convenience default is left-closed; the wire format still always + // carries an explicit "closed". + auto type = checked_pointer_cast(extension::range(int32())); + ASSERT_EQ(extension::RangeClosed::Left, type->closed()); + ASSERT_EQ(R"({"closed":"left"})", type->Serialize()); } // --------------------------------------------------------------------------- @@ -172,7 +172,17 @@ TEST(RangeType, Deserialize) { TEST(RangeType, DeserializeInvalidMetadata) { auto type = RangeInt32Right(); - // Empty string is valid (defaults to "right"); truly malformed JSON fails. + // "closed" is required on the wire: empty metadata is invalid. + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("empty string"), + type->Deserialize(type->storage_type(), "")); + + // A JSON object without the "closed" key is invalid. + EXPECT_RAISES_WITH_MESSAGE_THAT( + Invalid, testing::HasSubstr("missing the required \"closed\" key"), + type->Deserialize(type->storage_type(), "{}")); + + // Truly malformed JSON fails. EXPECT_RAISES_WITH_MESSAGE_THAT( Invalid, testing::HasSubstr("Missing a name for object member"), type->Deserialize(type->storage_type(), "{")); diff --git a/cpp/src/arrow/extension_type.cc b/cpp/src/arrow/extension_type.cc index 45cf5ed1e16f..1dd840621f20 100644 --- a/cpp/src/arrow/extension_type.cc +++ b/cpp/src/arrow/extension_type.cc @@ -157,7 +157,7 @@ static void CreateGlobalRegistry() { #ifdef ARROW_JSON ext_types.push_back(extension::fixed_shape_tensor(int64(), {})); ext_types.push_back(extension::opaque(null(), "", "")); - ext_types.push_back(extension::range(int32(), extension::RangeClosed::Right)); + ext_types.push_back(extension::range(int32())); ext_types.push_back(extension::variable_shape_tensor(int64(), 0)); #endif diff --git a/docs/source/format/CanonicalExtensions.rst b/docs/source/format/CanonicalExtensions.rst index 3e08ca87458b..16f1bdc13ffc 100644 --- a/docs/source/format/CanonicalExtensions.rst +++ b/docs/source/format/CanonicalExtensions.rst @@ -631,23 +631,25 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. * Description of the serialization: - The extension metadata **must** be either an empty string or a valid JSON - object. The JSON object may contain one key: + The extension metadata **must** be a valid JSON object containing the + **required** key: - * ``"closed"`` (string, optional): one of ``"left"``, ``"right"``, - ``"both"``, or ``"neither"``. When absent (including when the metadata is - an empty string), it defaults to ``"right"``. + * ``"closed"`` (string, **required**): one of ``"left"``, ``"right"``, + ``"both"``, or ``"neither"``. + + The closedness is **not** defaulted on the wire: an empty metadata string, + or a JSON object without a ``"closed"`` key, is invalid. This keeps the + serialized form unambiguous for consumers. Additional keys in the JSON object should be ignored to allow forward-compatible extensions. Examples: - - ``{"closed": "right"}`` -- half-open interval, right-closed (default) + - ``{"closed": "right"}`` -- half-open interval, right-closed - ``{"closed": "left"}`` -- half-open interval, left-closed - ``{"closed": "both"}`` -- closed interval - ``{"closed": "neither"}``-- open interval - - ``""`` -- equivalent to ``{"closed": "right"}`` * Semantics: From 31ee2f9bb62ed588c3032feadbdc6d1e9067e498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sat, 23 May 2026 21:40:35 +0000 Subject: [PATCH 03/13] test(format): fix arrow.range test compile and link errors Verified by building the arrow-canonical-extensions-test target (50/50 pass, 10/10 RangeType). Two fixes to the previously-uncompiled test: - include arrow/array/array_nested.h for the full StructArray definition (it is only forward-declared in type_fwd.h). - wrap the CheckDeserialize helper in an anonymous namespace to avoid a link-time collision with the identically named helper in opaque_test.cc. --- cpp/src/arrow/extension/range_test.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc index 2988439986c4..b9526024328a 100644 --- a/cpp/src/arrow/extension/range_test.cc +++ b/cpp/src/arrow/extension/range_test.cc @@ -18,6 +18,7 @@ #include #include +#include "arrow/array/array_nested.h" #include "arrow/extension/range.h" #include "arrow/extension_type.h" #include "arrow/io/memory.h" @@ -129,6 +130,8 @@ TEST(RangeType, CreateFromArray) { // --------------------------------------------------------------------------- // Deserialize - valid cases +namespace { + void CheckDeserialize(const std::string& serialized, const std::shared_ptr& expected) { auto type = checked_pointer_cast(expected); @@ -137,6 +140,8 @@ void CheckDeserialize(const std::string& serialized, ASSERT_EQ(*expected, *deserialized); } +} // namespace + TEST(RangeType, Deserialize) { // Normal JSON ASSERT_NO_FATAL_FAILURE( From b30c001c4bd3aa42dcf188bc859290a282dc1a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sun, 24 May 2026 00:01:29 +0000 Subject: [PATCH 04/13] style(format): align range extension type with neighbor C++ conventions --- cpp/src/arrow/extension/range.cc | 2 +- cpp/src/arrow/extension/range.h | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/src/arrow/extension/range.cc b/cpp/src/arrow/extension/range.cc index 97733fe04010..fe7089953932 100644 --- a/cpp/src/arrow/extension/range.cc +++ b/cpp/src/arrow/extension/range.cc @@ -78,7 +78,7 @@ std::shared_ptr RangeType::value_type() const { std::string RangeType::ToString(bool show_metadata) const { std::stringstream ss; - ss << "extension<" << extension_name() + ss << "extension<" << this->extension_name() << "[value_type=" << value_type()->ToString(show_metadata) << ", closed=" << ClosedToString(closed_) << "]>"; return ss.str(); diff --git a/cpp/src/arrow/extension/range.h b/cpp/src/arrow/extension/range.h index 3af60c1937ab..ab11734507b7 100644 --- a/cpp/src/arrow/extension/range.h +++ b/cpp/src/arrow/extension/range.h @@ -17,8 +17,6 @@ #pragma once -#include - #include "arrow/extension_type.h" #include "arrow/type.h" From 10d68d8ce6ee6999bf5c007bbace935dba2ae421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sun, 24 May 2026 00:01:29 +0000 Subject: [PATCH 05/13] docs(format): refine arrow.range spec and add status and C++ API entries --- docs/source/cpp/api/extension.rst | 14 +++++++ docs/source/format/CanonicalExtensions.rst | 47 ++++++++-------------- docs/source/status.rst | 2 + 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/docs/source/cpp/api/extension.rst b/docs/source/cpp/api/extension.rst index 5b9620907f2b..6b1e2e9a8df0 100644 --- a/docs/source/cpp/api/extension.rst +++ b/docs/source/cpp/api/extension.rst @@ -42,6 +42,10 @@ Extension Type classes :project: arrow_cpp :members: +.. doxygenclass:: arrow::extension::RangeType + :project: arrow_cpp + :members: + Extension Array classes ======================= @@ -61,3 +65,13 @@ Extension Array classes :project: arrow_cpp :members: +.. doxygenclass:: arrow::extension::RangeArray + :project: arrow_cpp + :members: + +Extension functions +=================== + +.. doxygenfunction:: arrow::extension::range + :project: arrow_cpp + diff --git a/docs/source/format/CanonicalExtensions.rst b/docs/source/format/CanonicalExtensions.rst index 16f1bdc13ffc..4d5a82adb62e 100644 --- a/docs/source/format/CanonicalExtensions.rst +++ b/docs/source/format/CanonicalExtensions.rst @@ -609,15 +609,14 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. fields share the same type T. The subtype is read directly from the storage struct and is **not** duplicated in the extension metadata. - The outer struct's validity bit marks a null/absent range (a missing range, - distinct from an empty range). - -.. note:: - - Both ``lower`` and ``upper`` struct fields **must** be nullable. A null - bound represents an infinite endpoint and is **always treated as - exclusive**, regardless of the value of the ``closed`` parameter. You - cannot include positive or negative infinity in a closed bound. + Both ``lower`` and ``upper`` fields **must** be nullable. A null bound + represents an infinite endpoint and is **always treated as exclusive**, + regardless of the value of the ``closed`` parameter; positive and negative + infinity can never be included in a closed bound. A null ``lower`` means + the range extends to negative infinity, a null ``upper`` means it extends to + positive infinity, and a range whose ``lower`` and ``upper`` are both null + is the universal range ``(-inf, +inf)``. The outer struct's validity bit + marks a null/absent range (a missing range, distinct from an empty range). * Extension type parameters: @@ -629,6 +628,12 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. * ``"both"`` -- both bounds inclusive: ``[lower, upper]`` * ``"neither"`` -- both bounds exclusive: ``(lower, upper)`` + A range thus contains every value x permitted by its finite bounds and + ``closed`` setting: with ``closed="both"`` every x such that + ``lower <= x <= upper``, with ``closed="neither"`` every x such that + ``lower < x < upper``. A range is *empty* when ``lower > upper``, or when + ``lower == upper`` and at least one bound is exclusive. + * Description of the serialization: The extension metadata **must** be a valid JSON object containing the @@ -639,10 +644,8 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. The closedness is **not** defaulted on the wire: an empty metadata string, or a JSON object without a ``"closed"`` key, is invalid. This keeps the - serialized form unambiguous for consumers. - - Additional keys in the JSON object should be ignored to allow - forward-compatible extensions. + serialized form unambiguous for consumers. Additional keys in the JSON + object should be ignored to allow forward-compatible extensions. Examples: @@ -651,24 +654,6 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. - ``{"closed": "both"}`` -- closed interval - ``{"closed": "neither"}``-- open interval -* Semantics: - - * A range value ``[lower, upper]`` with ``closed="both"`` contains every - value x such that ``lower <= x <= upper``. - * A range value ``[lower, upper]`` with ``closed="neither"`` is *empty* - when ``lower == upper`` (a degenerate open interval), and contains values - x such that ``lower < x < upper`` otherwise. - * Implementations should document behavior when ``lower > upper``; the - recommended interpretation is that such a range is *empty*. - * A null outer struct value represents a missing (absent) range, not an - empty range. - * A null ``lower`` field means the range has no lower bound (extends to - negative infinity); the lower bound is always exclusive in this case. - * A null ``upper`` field means the range has no upper bound (extends to - positive infinity); the upper bound is always exclusive in this case. - * A range where both ``lower`` and ``upper`` are null represents the - universal range ``(-inf, +inf)``. - .. _range types: https://www.postgresql.org/docs/current/rangetypes.html Community Extension Types diff --git a/docs/source/status.rst b/docs/source/status.rst index eba5395282cf..01cb40f093d5 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -133,6 +133,8 @@ Data Types +-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Timestamp With Offset | | | ✓ | | | ✓ | | | +-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ +| Range | ✓ | | | | | | | | ++-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ Notes: From 690fcf70cbfa747240b84a0452f1fe7cf19711d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sun, 24 May 2026 00:01:29 +0000 Subject: [PATCH 06/13] feat(python): add pyarrow bindings for arrow.range extension type --- docs/source/python/api/arrays.rst | 2 + docs/source/python/api/datatypes.rst | 2 + python/pyarrow/__init__.py | 9 +- python/pyarrow/array.pxi | 23 ++++ python/pyarrow/includes/libarrow.pxd | 21 ++++ python/pyarrow/lib.pxd | 4 + python/pyarrow/public-api.pxi | 2 + python/pyarrow/scalar.pxi | 6 + python/pyarrow/tests/test_extension_type.py | 66 +++++++++++ python/pyarrow/types.pxi | 123 ++++++++++++++++++++ 10 files changed, 254 insertions(+), 4 deletions(-) diff --git a/docs/source/python/api/arrays.rst b/docs/source/python/api/arrays.rst index 290ce09befb1..5a4dcecdc567 100644 --- a/docs/source/python/api/arrays.rst +++ b/docs/source/python/api/arrays.rst @@ -101,6 +101,7 @@ may expose data type-specific methods or properties. JsonArray UuidArray Bool8Array + RangeArray .. _api.scalar: @@ -169,3 +170,4 @@ classes may expose data type-specific methods or properties. JsonScalar UuidScalar Bool8Scalar + RangeScalar diff --git a/docs/source/python/api/datatypes.rst b/docs/source/python/api/datatypes.rst index ea9e547d32c7..71ee00557f33 100644 --- a/docs/source/python/api/datatypes.rst +++ b/docs/source/python/api/datatypes.rst @@ -73,6 +73,7 @@ These should be used to create Arrow data types and schemas. sparse_union opaque bool8 + range_ uuid json_ field @@ -146,6 +147,7 @@ implemented by PyArrow. JsonType UuidType Bool8Type + RangeType .. _api.types.checking: .. currentmodule:: pyarrow.types diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index adfc50d57395..1c4e82181aee 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -165,7 +165,7 @@ def print_entry(label, value): union, sparse_union, dense_union, dictionary, run_end_encoded, - bool8, fixed_shape_tensor, json_, opaque, uuid, + bool8, fixed_shape_tensor, json_, opaque, range_, uuid, field, type_for_alias, DataType, DictionaryType, StructType, @@ -177,7 +177,7 @@ def print_entry(label, value): Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, BaseExtensionType, ExtensionType, RunEndEncodedType, Bool8Type, FixedShapeTensorType, - JsonType, OpaqueType, UuidType, + JsonType, OpaqueType, RangeType, UuidType, UnknownExtensionType, register_extension_type, unregister_extension_type, DictionaryMemo, @@ -214,7 +214,7 @@ def print_entry(label, value): Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array, StructArray, ExtensionArray, RunEndEncodedArray, Bool8Array, FixedShapeTensorArray, - JsonArray, OpaqueArray, UuidArray, + JsonArray, OpaqueArray, RangeArray, UuidArray, scalar, NA, _NULL as NULL, Scalar, NullScalar, BooleanScalar, Int8Scalar, Int16Scalar, Int32Scalar, Int64Scalar, @@ -232,7 +232,8 @@ def print_entry(label, value): FixedSizeBinaryScalar, DictionaryScalar, MapScalar, StructScalar, UnionScalar, RunEndEncodedScalar, Bool8Scalar, ExtensionScalar, - FixedShapeTensorScalar, JsonScalar, OpaqueScalar, UuidScalar) + FixedShapeTensorScalar, JsonScalar, OpaqueScalar, + RangeScalar, UuidScalar) # Buffers, allocation from pyarrow.lib import (DeviceAllocationType, Device, MemoryManager, diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 3060c533255b..9b2646f04c5b 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -5314,6 +5314,29 @@ cdef class Bool8Array(ExtensionArray): return Bool8Array.from_storage(storage_arr) +cdef class RangeArray(ExtensionArray): + """ + Concrete class for range extension arrays. + + Examples + -------- + Define the extension type for a range array + + >>> import pyarrow as pa + >>> range_type = pa.range_(pa.int32(), "both") + + Create an extension array + + >>> storage = pa.array( + ... [{"lower": 1, "upper": 5}, {"lower": None, "upper": 10}], + ... range_type.storage_type, + ... ) + >>> arr = pa.ExtensionArray.from_storage(range_type, storage) + >>> isinstance(arr, pa.RangeArray) + True + """ + + cdef dict _array_classes = { _Type_NA: NullArray, _Type_BOOL: BooleanArray, diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 16784b51ecc7..4b2a011dfffb 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3134,6 +3134,27 @@ cdef extern from "arrow/extension/bool8.h" namespace "arrow::extension" nogil: cdef cppclass CBool8Array" arrow::extension::Bool8Array"(CExtensionArray): pass + +cdef extern from "arrow/extension/range.h" namespace "arrow::extension" nogil: + cdef enum class CRangeClosed" arrow::extension::RangeClosed": + Left + Right + Both + Neither + + cdef cppclass CRangeType" arrow::extension::RangeType"(CExtensionType): + + @staticmethod + CResult[shared_ptr[CDataType]] Make(shared_ptr[CDataType] value_type, + CRangeClosed closed) + + CRangeClosed closed() + shared_ptr[CDataType] value_type() + + cdef cppclass CRangeArray" arrow::extension::RangeArray"(CExtensionArray): + pass + + cdef extern from "arrow/util/compression.h" namespace "arrow" nogil: cdef enum CCompressionType" arrow::Compression::type": CCompressionType_UNCOMPRESSED" arrow::Compression::UNCOMPRESSED" diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 38f1ac69a807..d1226699ca86 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -203,6 +203,10 @@ cdef class Bool8Type(BaseExtensionType): cdef: const CBool8Type* bool8_ext_type +cdef class RangeType(BaseExtensionType): + cdef: + const CRangeType* range_ext_type + cdef class OpaqueType(BaseExtensionType): cdef: const COpaqueType* opaque_ext_type diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi index d1fa1192debc..7bb56435b7e2 100644 --- a/python/pyarrow/public-api.pxi +++ b/python/pyarrow/public-api.pxi @@ -131,6 +131,8 @@ cdef api object pyarrow_wrap_data_type( out = Bool8Type.__new__(Bool8Type) elif extension_name == b"arrow.fixed_shape_tensor": out = FixedShapeTensorType.__new__(FixedShapeTensorType) + elif extension_name == b"arrow.range": + out = RangeType.__new__(RangeType) elif extension_name == b"arrow.opaque": out = OpaqueType.__new__(OpaqueType) elif extension_name == b"arrow.uuid": diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi index 863ab2b66a8c..86546e2a604d 100644 --- a/python/pyarrow/scalar.pxi +++ b/python/pyarrow/scalar.pxi @@ -1640,6 +1640,12 @@ cdef class Bool8Scalar(ExtensionScalar): py_val = super().as_py() return None if py_val is None else py_val != 0 + +cdef class RangeScalar(ExtensionScalar): + """ + Concrete class for range extension scalar. + """ + cdef dict _scalar_classes = { _Type_BOOL: BooleanScalar, _Type_UINT8: UInt8Scalar, diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py index a9d15cfec10a..ad2ca0721606 100644 --- a/python/pyarrow/tests/test_extension_type.py +++ b/python/pyarrow/tests/test_extension_type.py @@ -2080,6 +2080,72 @@ def test_opaque_type(pickle_module, storage_type, storage): assert inner == storage +@pytest.mark.parametrize("closed", ["left", "right", "both", "neither"]) +@pytest.mark.parametrize("value_type,bounds", [ + (pa.int32(), [{"lower": 1, "upper": 5}, {"lower": None, "upper": 10}]), + (pa.int64(), [{"lower": None, "upper": None}, {"lower": 2, "upper": 8}]), + (pa.float64(), [{"lower": 0.0, "upper": 1.5}, None]), +]) +def test_range_type(pickle_module, closed, value_type, bounds): + range_type = pa.range_(value_type, closed) + assert range_type.extension_name == "arrow.range" + assert range_type.value_type == value_type + assert range_type.closed == closed + assert range_type.storage_type == pa.struct([ + pa.field("lower", value_type, nullable=True), + pa.field("upper", value_type, nullable=True), + ]) + assert "arrow.range" in str(range_type) + + # the closed parameter defaults to "left" + assert pa.range_(value_type).closed == "left" + + assert range_type == range_type + assert range_type == pa.range_(value_type, closed) + assert range_type != value_type + # different closed parameter -> not equal + other_closed = "right" if closed != "right" else "left" + assert range_type != pa.range_(value_type, other_closed) + # different value type -> not equal + assert range_type != pa.range_(pa.decimal128(12, 3), closed) + + # Pickle roundtrip + result = pickle_module.loads(pickle_module.dumps(range_type)) + assert result == range_type + assert result.closed == closed + assert result.value_type == value_type + + # IPC roundtrip + range_arr_class = range_type.__arrow_ext_class__() + storage = pa.array(bounds, range_type.storage_type) + arr = pa.ExtensionArray.from_storage(range_type, storage) + assert isinstance(arr, range_arr_class) + + # extension is registered by default + buf = ipc_write_batch(pa.RecordBatch.from_arrays([arr], ["ext"])) + batch = ipc_read_batch(buf) + + assert batch.column(0).type.extension_name == "arrow.range" + assert batch.column(0).type.closed == closed + assert isinstance(batch.column(0), range_arr_class) + assert batch.column(0) == arr + + # cast storage -> extension type + result = storage.cast(range_type) + assert result == arr + + # cast extension type -> storage type + inner = arr.cast(range_type.storage_type) + assert inner == storage + + +def test_range_type_invalid_closed(): + with pytest.raises(ValueError, match="Invalid value for range"): + pa.range_(pa.int32(), "invalid") + with pytest.raises(ValueError, match="Invalid value for range"): + pa.range_(pa.int32(), "") + + def test_bool8_type(pickle_module): bool8_type = pa.bool8() storage_type = pa.int8() diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index f9530a34362f..13fca0f6de3e 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -2088,6 +2088,62 @@ cdef class Bool8Type(BaseExtensionType): return Bool8Scalar +cdef class RangeType(BaseExtensionType): + """ + Concrete class for range extension type. + + Range represents a bounded set (a mathematical interval) over an orderable + Arrow value type. The underlying storage is a Struct with two nullable + fields "lower" and "upper" of the value type, where a null bound denotes an + unbounded (infinite) side. The "closed" parameter controls which finite + bounds are inclusive. + + Examples + -------- + Create an instance of range extension type: + + >>> import pyarrow as pa + >>> pa.range_(pa.int32(), "both") + RangeType(extension) + """ + + cdef void init(self, const shared_ptr[CDataType]& type) except *: + BaseExtensionType.init(self, type) + self.range_ext_type = type.get() + + @property + def value_type(self): + """ + The Arrow value type of the "lower" and "upper" bounds. + """ + return pyarrow_wrap_data_type(self.range_ext_type.value_type()) + + @property + def closed(self): + """ + Which bound(s) are inclusive, as one of "left", "right", "both" or + "neither". + """ + cdef CRangeClosed c_closed = self.range_ext_type.closed() + if c_closed == CRangeClosed.Left: + return "left" + elif c_closed == CRangeClosed.Right: + return "right" + elif c_closed == CRangeClosed.Both: + return "both" + else: + return "neither" + + def __arrow_ext_class__(self): + return RangeArray + + def __reduce__(self): + return range_, (self.value_type, self.closed) + + def __arrow_ext_scalar_class__(self): + return RangeScalar + + cdef class OpaqueType(BaseExtensionType): """ Concrete class for opaque extension type. @@ -5703,6 +5759,73 @@ def bool8(): return out +def range_(DataType value_type not None, str closed="left"): + """ + Create instance of range extension type. + + Parameters + ---------- + value_type : DataType + The orderable Arrow type of the "lower" and "upper" interval bounds. + closed : str, default "left" + Which bound(s) are inclusive. One of "left", "right", "both" or + "neither". + + Examples + -------- + Create an instance of a range extension type: + + >>> import pyarrow as pa + >>> type = pa.range_(pa.int32(), "both") + >>> type + RangeType(extension) + + Inspect the data type: + + >>> type.value_type + DataType(int32) + >>> type.closed + 'both' + >>> type.storage_type + StructType(struct) + + Create a range array: + + >>> storage = pa.array( + ... [{"lower": 1, "upper": 5}, {"lower": None, "upper": 10}], + ... type.storage_type, + ... ) + >>> arr = pa.ExtensionArray.from_storage(type, storage) + >>> arr.type + RangeType(extension) + + Returns + ------- + type : RangeType + """ + + cdef CRangeClosed c_closed + if closed == "left": + c_closed = CRangeClosed.Left + elif closed == "right": + c_closed = CRangeClosed.Right + elif closed == "both": + c_closed = CRangeClosed.Both + elif closed == "neither": + c_closed = CRangeClosed.Neither + else: + raise ValueError( + f"Invalid value for range \"closed\" parameter: {closed!r}. " + "Expected one of: 'left', 'right', 'both', 'neither'.") + + cdef: + shared_ptr[CDataType] c_type = GetResultValue( + CRangeType.Make(value_type.sp_type, c_closed)) + RangeType out = RangeType.__new__(RangeType) + out.init(c_type) + return out + + def opaque(DataType storage_type, str type_name not None, str vendor_name not None): """ Create instance of opaque extension type. From b3253dfdfb47057cc5489a444352b9963b14ebcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sun, 24 May 2026 12:30:23 +0000 Subject: [PATCH 07/13] feat(format): allow non-nullable bounds in arrow.range --- cpp/src/arrow/extension/range.cc | 24 +++++------ cpp/src/arrow/extension/range.h | 21 +++++++--- cpp/src/arrow/extension/range_test.cc | 47 ++++++++++++++++------ docs/source/format/CanonicalExtensions.rst | 29 +++++++------ 4 files changed, 77 insertions(+), 44 deletions(-) diff --git a/cpp/src/arrow/extension/range.cc b/cpp/src/arrow/extension/range.cc index fe7089953932..d39d90acfe96 100644 --- a/cpp/src/arrow/extension/range.cc +++ b/cpp/src/arrow/extension/range.cc @@ -60,10 +60,12 @@ Result ClosedFromString(std::string_view s) { } /// Build the storage Struct type for a given value subtype. -std::shared_ptr MakeStorageType(const std::shared_ptr& value_type) { - // Both "lower" and "upper" are nullable (null = infinite bound). - return struct_({field("lower", value_type, /*nullable=*/true), - field("upper", value_type, /*nullable=*/true)}); +std::shared_ptr MakeStorageType(const std::shared_ptr& value_type, + bool allow_unbounded) { + // Nullable bounds can represent an unbounded (infinite) endpoint; non-nullable + // bounds are always finite. + return struct_({field("lower", value_type, allow_unbounded), + field("upper", value_type, allow_unbounded)}); } } // namespace @@ -135,12 +137,6 @@ Result> RangeType::Deserialize( "RangeType storage Struct field 1 must be named \"upper\", got \"", upper_field->name(), "\""); } - if (!lower_field->nullable()) { - return Status::Invalid("RangeType storage Struct field \"lower\" must be nullable"); - } - if (!upper_field->nullable()) { - return Status::Invalid("RangeType storage Struct field \"upper\" must be nullable"); - } if (!lower_field->type()->Equals(*upper_field->type())) { return Status::Invalid( "RangeType storage Struct fields \"lower\" and \"upper\" must have the same " @@ -191,8 +187,8 @@ std::shared_ptr RangeType::MakeArray(std::shared_ptr data) con } Result> RangeType::Make( - std::shared_ptr value_type, RangeClosed closed) { - auto storage = MakeStorageType(value_type); + std::shared_ptr value_type, RangeClosed closed, bool allow_unbounded) { + auto storage = MakeStorageType(value_type, allow_unbounded); return std::make_shared(std::move(storage), closed); } @@ -200,8 +196,8 @@ Result> RangeType::Make( // Free factory function std::shared_ptr range(std::shared_ptr value_type, - RangeClosed closed) { - auto result = RangeType::Make(std::move(value_type), closed); + RangeClosed closed, bool allow_unbounded) { + auto result = RangeType::Make(std::move(value_type), closed, allow_unbounded); ARROW_CHECK_OK(result.status()); return std::move(result).ValueOrDie(); } diff --git a/cpp/src/arrow/extension/range.h b/cpp/src/arrow/extension/range.h index ab11734507b7..b9f5f55e6106 100644 --- a/cpp/src/arrow/extension/range.h +++ b/cpp/src/arrow/extension/range.h @@ -39,9 +39,12 @@ enum class RangeClosed { /// \brief RangeType represents a bounded set (mathematical interval) over an /// orderable Arrow type T. /// -/// Storage is a Struct with exactly two nullable fields: -/// - "lower": T NULLABLE (null = unbounded below, i.e. -infinity) -/// - "upper": T NULLABLE (null = unbounded above, i.e. +infinity) +/// Storage is a Struct with exactly two fields "lower" and "upper" of the same +/// orderable type T. Each field may independently be nullable or not: a nullable +/// bound can hold null to represent an unbounded (infinite) endpoint on that +/// side, while a non-nullable bound is always finite. +/// - "lower": T (null, when nullable = unbounded below, i.e. -infinity) +/// - "upper": T (null, when nullable = unbounded above, i.e. +infinity) /// /// The outer struct's validity bit marks a null/absent range. /// @@ -70,11 +73,16 @@ class ARROW_EXPORT RangeType : public ExtensionType { /// \brief Factory function. /// - /// Constructs the required two-field struct storage type internally. + /// Constructs the two-field struct storage type internally. /// \param[in] value_type The orderable Arrow subtype T for lower and upper. /// \param[in] closed Which bound(s) are inclusive. + /// \param[in] allow_unbounded Whether each side may be unbounded (infinite). + /// When true, the "lower" and "upper" fields are nullable and a null bound + /// denotes an infinite endpoint; when false, both bounds are non-nullable + /// and the range is always finite. Defaults to true. static Result> Make(std::shared_ptr value_type, - RangeClosed closed = RangeClosed::Left); + RangeClosed closed = RangeClosed::Left, + bool allow_unbounded = true); /// \brief Return the bound-inclusivity parameter. RangeClosed closed() const { return closed_; } @@ -97,6 +105,7 @@ class ARROW_EXPORT RangeArray : public ExtensionArray { /// This is a convenience wrapper around RangeType::Make that aborts on error. /// For recoverable error handling prefer RangeType::Make. ARROW_EXPORT std::shared_ptr range(std::shared_ptr value_type, - RangeClosed closed = RangeClosed::Left); + RangeClosed closed = RangeClosed::Left, + bool allow_unbounded = true); } // namespace arrow::extension diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc index b9526024328a..ef4463fb0c11 100644 --- a/cpp/src/arrow/extension/range_test.cc +++ b/cpp/src/arrow/extension/range_test.cc @@ -238,20 +238,43 @@ TEST(RangeType, DeserializeInvalidStorage) { EXPECT_RAISES_WITH_MESSAGE_THAT( Invalid, testing::HasSubstr("same type"), type->Deserialize(mismatched_types, R"({"closed":"right"})")); +} - // Non-nullable lower field. - auto lower_not_nullable = - struct_({field("lower", int32(), /*nullable=*/false), field("upper", int32(), true)}); - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, testing::HasSubstr("\"lower\" must be nullable"), - type->Deserialize(lower_not_nullable, R"({"closed":"right"})")); +// --------------------------------------------------------------------------- +// Non-nullable / asymmetric bounds +// +// Bound nullability is only needed to represent an unbounded (infinite) +// endpoint; non-nullable bounds describe a finite-only range and are accepted. - // Non-nullable upper field. - auto upper_not_nullable = - struct_({field("lower", int32(), true), field("upper", int32(), /*nullable=*/false)}); - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, testing::HasSubstr("\"upper\" must be nullable"), - type->Deserialize(upper_not_nullable, R"({"closed":"right"})")); +TEST(RangeType, NonNullableBounds) { + auto type = RangeInt32Right(); + + // Both bounds non-nullable: accepted (a finite-only range). + auto both_non_nullable = struct_( + {field("lower", int32(), /*nullable=*/false), + field("upper", int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN(auto from_non_nullable, + type->Deserialize(both_non_nullable, R"({"closed":"right"})")); + ASSERT_EQ( + *int32(), + *checked_pointer_cast(from_non_nullable)->value_type()); + + // Asymmetric: lower nullable (may be -inf), upper non-nullable (always finite). + auto asymmetric = struct_( + {field("lower", int32(), /*nullable=*/true), + field("upper", int32(), /*nullable=*/false)}); + ASSERT_OK_AND_ASSIGN(auto from_asymmetric, + type->Deserialize(asymmetric, R"({"closed":"left"})")); + ASSERT_EQ(extension::RangeClosed::Left, + checked_pointer_cast(from_asymmetric)->closed()); + + // The factory can build non-nullable bounds via allow_unbounded=false. + auto finite = checked_pointer_cast( + extension::range(int32(), extension::RangeClosed::Both, /*allow_unbounded=*/false)); + const auto& finite_storage = + internal::checked_cast(*finite->storage_type()); + ASSERT_FALSE(finite_storage.field(0)->nullable()); + ASSERT_FALSE(finite_storage.field(1)->nullable()); } // --------------------------------------------------------------------------- diff --git a/docs/source/format/CanonicalExtensions.rst b/docs/source/format/CanonicalExtensions.rst index 4d5a82adb62e..40ff32db79d8 100644 --- a/docs/source/format/CanonicalExtensions.rst +++ b/docs/source/format/CanonicalExtensions.rst @@ -599,24 +599,29 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. * The storage type of the extension is a ``Struct`` with exactly **two fields, in order**: - * ``lower``: the lower bound, type **T**, **nullable**. - A null value means the range is unbounded below (negative infinity). - * ``upper``: the upper bound, type **T**, **nullable**. - A null value means the range is unbounded above (positive infinity). + * ``lower``: the lower bound, type **T**, *optionally nullable*. + When the field is nullable, a null value means the range is unbounded below + (negative infinity). + * ``upper``: the upper bound, type **T**, *optionally nullable*. + When the field is nullable, a null value means the range is unbounded above + (positive infinity). **T** (the *subtype* or *value type*) may be any orderable Arrow type: integer, floating-point, decimal, date, time, or timestamp types. Both fields share the same type T. The subtype is read directly from the storage struct and is **not** duplicated in the extension metadata. - Both ``lower`` and ``upper`` fields **must** be nullable. A null bound - represents an infinite endpoint and is **always treated as exclusive**, - regardless of the value of the ``closed`` parameter; positive and negative - infinity can never be included in a closed bound. A null ``lower`` means - the range extends to negative infinity, a null ``upper`` means it extends to - positive infinity, and a range whose ``lower`` and ``upper`` are both null - is the universal range ``(-inf, +inf)``. The outer struct's validity bit - marks a null/absent range (a missing range, distinct from an empty range). + Each of ``lower`` and ``upper`` **may** be nullable, independently of the + other. Nullability is **only** needed to represent an unbounded side: a + nullable bound may hold null to mean an infinite endpoint, while a + non-nullable bound is always finite. A null bound is **always treated as + exclusive**, regardless of the value of the ``closed`` parameter; positive and + negative infinity can never be included in a closed bound. A null ``lower`` + means the range extends to negative infinity, a null ``upper`` means it + extends to positive infinity, and a range whose ``lower`` and ``upper`` are + both null (and both nullable) is the universal range ``(-inf, +inf)``. The + outer struct's validity bit marks a null/absent range (a missing range, + distinct from an empty range). * Extension type parameters: From 64e108840c8bf51b468dfb2a269ddeb1292a23ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sun, 24 May 2026 12:30:23 +0000 Subject: [PATCH 08/13] feat(python): add allow_unbounded option to pyarrow range_ --- python/pyarrow/includes/libarrow.pxd | 3 ++- python/pyarrow/tests/test_extension_type.py | 22 +++++++++++++++++++++ python/pyarrow/types.pxi | 17 ++++++++++------ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 4b2a011dfffb..aa647f7f607a 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3146,7 +3146,8 @@ cdef extern from "arrow/extension/range.h" namespace "arrow::extension" nogil: @staticmethod CResult[shared_ptr[CDataType]] Make(shared_ptr[CDataType] value_type, - CRangeClosed closed) + CRangeClosed closed, + c_bool allow_unbounded) CRangeClosed closed() shared_ptr[CDataType] value_type() diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py index ad2ca0721606..41d33ff70152 100644 --- a/python/pyarrow/tests/test_extension_type.py +++ b/python/pyarrow/tests/test_extension_type.py @@ -2146,6 +2146,28 @@ def test_range_type_invalid_closed(): pa.range_(pa.int32(), "") +def test_range_type_allow_unbounded(): + # Default: bounds are nullable (can represent an unbounded / infinite side). + nullable = pa.range_(pa.int32(), "both") + assert nullable.storage_type.field("lower").nullable + assert nullable.storage_type.field("upper").nullable + + # allow_unbounded=False: a finite-only range with non-nullable bounds. + finite = pa.range_(pa.int32(), "both", allow_unbounded=False) + assert not finite.storage_type.field("lower").nullable + assert not finite.storage_type.field("upper").nullable + assert finite.value_type == pa.int32() + assert finite.closed == "both" + + # Distinct types: storage nullability differs. + assert finite != nullable + + # A non-nullable-bounds range round-trips through its storage. + storage = pa.array([{"lower": 1, "upper": 5}], finite.storage_type) + arr = pa.ExtensionArray.from_storage(finite, storage) + assert arr.type == finite + + def test_bool8_type(pickle_module): bool8_type = pa.bool8() storage_type = pa.int8() diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 13fca0f6de3e..c9b9faaad2d1 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -2093,10 +2093,10 @@ cdef class RangeType(BaseExtensionType): Concrete class for range extension type. Range represents a bounded set (a mathematical interval) over an orderable - Arrow value type. The underlying storage is a Struct with two nullable - fields "lower" and "upper" of the value type, where a null bound denotes an - unbounded (infinite) side. The "closed" parameter controls which finite - bounds are inclusive. + Arrow value type. The underlying storage is a Struct with two fields + "lower" and "upper" of the value type, each optionally nullable; when a + bound field is nullable, a null value denotes an unbounded (infinite) side. + The "closed" parameter controls which finite bounds are inclusive. Examples -------- @@ -5759,7 +5759,7 @@ def bool8(): return out -def range_(DataType value_type not None, str closed="left"): +def range_(DataType value_type not None, str closed="left", allow_unbounded=True): """ Create instance of range extension type. @@ -5770,6 +5770,11 @@ def range_(DataType value_type not None, str closed="left"): closed : str, default "left" Which bound(s) are inclusive. One of "left", "right", "both" or "neither". + allow_unbounded : bool, default True + Whether each side may be unbounded (infinite). When True the "lower" and + "upper" storage fields are nullable (a null bound is an infinite + endpoint); when False both bounds are non-nullable and the range is + always finite. Examples -------- @@ -5820,7 +5825,7 @@ def range_(DataType value_type not None, str closed="left"): cdef: shared_ptr[CDataType] c_type = GetResultValue( - CRangeType.Make(value_type.sp_type, c_closed)) + CRangeType.Make(value_type.sp_type, c_closed, allow_unbounded)) RangeType out = RangeType.__new__(RangeType) out.init(c_type) return out From 65e19d40935d10155ff3f4c147db9cc8f8344eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sun, 24 May 2026 22:55:41 +0000 Subject: [PATCH 09/13] style(format): apply clang-format and cmake-format to range extension --- cpp/src/arrow/extension/CMakeLists.txt | 5 +- cpp/src/arrow/extension/range.cc | 30 +++++------ cpp/src/arrow/extension/range_test.cc | 72 ++++++++++++-------------- 3 files changed, 50 insertions(+), 57 deletions(-) diff --git a/cpp/src/arrow/extension/CMakeLists.txt b/cpp/src/arrow/extension/CMakeLists.txt index 966c927bf2ce..8f7bae112e9f 100644 --- a/cpp/src/arrow/extension/CMakeLists.txt +++ b/cpp/src/arrow/extension/CMakeLists.txt @@ -18,7 +18,10 @@ set(CANONICAL_EXTENSION_TESTS bool8_test.cc json_test.cc uuid_test.cc) if(ARROW_JSON) - list(APPEND CANONICAL_EXTENSION_TESTS tensor_extension_array_test.cc opaque_test.cc + list(APPEND + CANONICAL_EXTENSION_TESTS + tensor_extension_array_test.cc + opaque_test.cc range_test.cc) endif() diff --git a/cpp/src/arrow/extension/range.cc b/cpp/src/arrow/extension/range.cc index d39d90acfe96..466951c961fc 100644 --- a/cpp/src/arrow/extension/range.cc +++ b/cpp/src/arrow/extension/range.cc @@ -101,9 +101,8 @@ std::string RangeType::Serialize() const { rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); auto closed_str = ClosedToString(closed_); - rapidjson::Value closed_value(closed_str.data(), - static_cast(closed_str.size()), - allocator); + rapidjson::Value closed_value( + closed_str.data(), static_cast(closed_str.size()), allocator); document.AddMember(rapidjson::Value("closed", allocator), closed_value, allocator); rapidjson::StringBuffer buffer; @@ -121,9 +120,8 @@ Result> RangeType::Deserialize( } const auto& struct_type = internal::checked_cast(*storage_type); if (struct_type.num_fields() != 2) { - return Status::Invalid( - "RangeType storage Struct must have exactly 2 fields, got ", - struct_type.num_fields()); + return Status::Invalid("RangeType storage Struct must have exactly 2 fields, got ", + struct_type.num_fields()); } const auto& lower_field = struct_type.field(0); const auto& upper_field = struct_type.field(1); @@ -163,18 +161,17 @@ Result> RangeType::Deserialize( return Status::Invalid("Invalid serialized JSON data for RangeType: not an object"); } if (!document.HasMember("closed")) { - return Status::Invalid( - "RangeType metadata is missing the required \"closed\" key: ", serialized_data); + return Status::Invalid("RangeType metadata is missing the required \"closed\" key: ", + serialized_data); } const auto& closed_val = document["closed"]; if (!closed_val.IsString()) { return Status::Invalid( "Invalid serialized JSON data for RangeType: \"closed\" is not a string"); } - ARROW_ASSIGN_OR_RAISE( - RangeClosed closed, - ClosedFromString( - std::string_view(closed_val.GetString(), closed_val.GetStringLength()))); + ARROW_ASSIGN_OR_RAISE(RangeClosed closed, + ClosedFromString(std::string_view(closed_val.GetString(), + closed_val.GetStringLength()))); return std::make_shared(std::move(storage_type), closed); } @@ -186,8 +183,9 @@ std::shared_ptr RangeType::MakeArray(std::shared_ptr data) con return std::make_shared(data); } -Result> RangeType::Make( - std::shared_ptr value_type, RangeClosed closed, bool allow_unbounded) { +Result> RangeType::Make(std::shared_ptr value_type, + RangeClosed closed, + bool allow_unbounded) { auto storage = MakeStorageType(value_type, allow_unbounded); return std::make_shared(std::move(storage), closed); } @@ -195,8 +193,8 @@ Result> RangeType::Make( // --------------------------------------------------------------------------- // Free factory function -std::shared_ptr range(std::shared_ptr value_type, - RangeClosed closed, bool allow_unbounded) { +std::shared_ptr range(std::shared_ptr value_type, RangeClosed closed, + bool allow_unbounded) { auto result = RangeType::Make(std::move(value_type), closed, allow_unbounded); ARROW_CHECK_OK(result.status()); return std::move(result).ValueOrDie(); diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc index ef4463fb0c11..4b18327e42e1 100644 --- a/cpp/src/arrow/extension/range_test.cc +++ b/cpp/src/arrow/extension/range_test.cc @@ -69,14 +69,14 @@ TEST(RangeType, Basics) { TEST(RangeType, AllClosedValues) { using C = extension::RangeClosed; - auto left = checked_pointer_cast( - extension::range(int32(), C::Left)); - auto right = checked_pointer_cast( - extension::range(int32(), C::Right)); - auto both = checked_pointer_cast( - extension::range(int32(), C::Both)); - auto neither = checked_pointer_cast( - extension::range(int32(), C::Neither)); + auto left = + checked_pointer_cast(extension::range(int32(), C::Left)); + auto right = + checked_pointer_cast(extension::range(int32(), C::Right)); + auto both = + checked_pointer_cast(extension::range(int32(), C::Both)); + auto neither = + checked_pointer_cast(extension::range(int32(), C::Neither)); ASSERT_EQ(R"({"closed":"left"})", left->Serialize()); ASSERT_EQ(R"({"closed":"right"})", right->Serialize()); @@ -119,9 +119,9 @@ TEST(RangeType, CreateFromArray) { auto storage_type = type->storage_type(); auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); - ASSERT_OK_AND_ASSIGN(auto storage, StructArray::Make({lower, upper}, - {field("lower", int32(), true), - field("upper", int32(), true)})); + ASSERT_OK_AND_ASSIGN( + auto storage, StructArray::Make({lower, upper}, {field("lower", int32(), true), + field("upper", int32(), true)})); auto array = ExtensionType::WrapArray(type, storage); ASSERT_EQ(3, array->length()); ASSERT_EQ(0, array->null_count()); @@ -147,12 +147,10 @@ TEST(RangeType, Deserialize) { ASSERT_NO_FATAL_FAILURE( CheckDeserialize(R"({"closed": "right"})", extension::range(int32(), extension::RangeClosed::Right))); - ASSERT_NO_FATAL_FAILURE( - CheckDeserialize(R"({"closed": "left"})", - extension::range(int32(), extension::RangeClosed::Left))); - ASSERT_NO_FATAL_FAILURE( - CheckDeserialize(R"({"closed": "both"})", - extension::range(int32(), extension::RangeClosed::Both))); + ASSERT_NO_FATAL_FAILURE(CheckDeserialize( + R"({"closed": "left"})", extension::range(int32(), extension::RangeClosed::Left))); + ASSERT_NO_FATAL_FAILURE(CheckDeserialize( + R"({"closed": "both"})", extension::range(int32(), extension::RangeClosed::Both))); ASSERT_NO_FATAL_FAILURE( CheckDeserialize(R"({"closed": "neither"})", extension::range(int32(), extension::RangeClosed::Neither))); @@ -178,9 +176,8 @@ TEST(RangeType, DeserializeInvalidMetadata) { auto type = RangeInt32Right(); // "closed" is required on the wire: empty metadata is invalid. - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, testing::HasSubstr("empty string"), - type->Deserialize(type->storage_type(), "")); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("empty string"), + type->Deserialize(type->storage_type(), "")); // A JSON object without the "closed" key is invalid. EXPECT_RAISES_WITH_MESSAGE_THAT( @@ -188,9 +185,9 @@ TEST(RangeType, DeserializeInvalidMetadata) { type->Deserialize(type->storage_type(), "{}")); // Truly malformed JSON fails. - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, testing::HasSubstr("Missing a name for object member"), - type->Deserialize(type->storage_type(), "{")); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, + testing::HasSubstr("Missing a name for object member"), + type->Deserialize(type->storage_type(), "{")); EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("not an object"), type->Deserialize(type->storage_type(), "[]")); @@ -214,9 +211,8 @@ TEST(RangeType, DeserializeInvalidStorage) { // Wrong number of fields. auto one_field = struct_({field("lower", int32(), true)}); - EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, testing::HasSubstr("exactly 2 fields"), - type->Deserialize(one_field, R"({"closed":"right"})")); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("exactly 2 fields"), + type->Deserialize(one_field, R"({"closed":"right"})")); // Wrong field name for field 0. auto bad_lower_name = @@ -250,19 +246,16 @@ TEST(RangeType, NonNullableBounds) { auto type = RangeInt32Right(); // Both bounds non-nullable: accepted (a finite-only range). - auto both_non_nullable = struct_( - {field("lower", int32(), /*nullable=*/false), - field("upper", int32(), /*nullable=*/false)}); + auto both_non_nullable = struct_({field("lower", int32(), /*nullable=*/false), + field("upper", int32(), /*nullable=*/false)}); ASSERT_OK_AND_ASSIGN(auto from_non_nullable, type->Deserialize(both_non_nullable, R"({"closed":"right"})")); - ASSERT_EQ( - *int32(), - *checked_pointer_cast(from_non_nullable)->value_type()); + ASSERT_EQ(*int32(), + *checked_pointer_cast(from_non_nullable)->value_type()); // Asymmetric: lower nullable (may be -inf), upper non-nullable (always finite). - auto asymmetric = struct_( - {field("lower", int32(), /*nullable=*/true), - field("upper", int32(), /*nullable=*/false)}); + auto asymmetric = struct_({field("lower", int32(), /*nullable=*/true), + field("upper", int32(), /*nullable=*/false)}); ASSERT_OK_AND_ASSIGN(auto from_asymmetric, type->Deserialize(asymmetric, R"({"closed":"left"})")); ASSERT_EQ(extension::RangeClosed::Left, @@ -301,12 +294,11 @@ TEST(RangeType, BatchRoundTrip) { auto type = RangeInt32Right(); auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); - ASSERT_OK_AND_ASSIGN(auto storage, StructArray::Make({lower, upper}, - {field("lower", int32(), true), - field("upper", int32(), true)})); + ASSERT_OK_AND_ASSIGN( + auto storage, StructArray::Make({lower, upper}, {field("lower", int32(), true), + field("upper", int32(), true)})); auto array = ExtensionType::WrapArray(type, storage); - auto batch = - RecordBatch::Make(schema({field("rng", type)}), array->length(), {array}); + auto batch = RecordBatch::Make(schema({field("rng", type)}), array->length(), {array}); std::shared_ptr written; { From 0eade001d021cb3fb85ba11b2052728baeb35eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Thu, 4 Jun 2026 19:14:25 +0000 Subject: [PATCH 10/13] feat(format): add arrow.range_inc canonical extension type Add a sibling canonical extension type to arrow.range that stores bound inclusivity per value via non-nullable boolean lower_inc/upper_inc fields, storage Struct. arrow.range carries a single type-level closed parameter, sufficient for discrete ranges that canonicalize to one closedness (int4range, int8range, daterange). Continuous ranges (numrange, tsrange, tstzrange) cannot be canonicalized, so closedness must travel with each value. arrow.range_inc mirrors PostgreSQL's internal range representation for that case; both types coexist. The type has no metadata parameters: inclusivity lives in storage, so Serialize emits {} and Deserialize accepts empty/{}/extra keys. A null (infinite) bound is always exclusive regardless of its flag. Covers C++ (type, array, registration, tests), pyarrow bindings and tests, and the format spec, status table, and C++/Python API docs. --- cpp/src/arrow/extension/range.cc | 142 ++++++++++++ cpp/src/arrow/extension/range.h | 73 +++++++ cpp/src/arrow/extension/range_test.cc | 231 ++++++++++++++++++++ cpp/src/arrow/extension_type.cc | 1 + docs/source/cpp/api/extension.rst | 11 + docs/source/format/CanonicalExtensions.rst | 73 +++++++ docs/source/python/api/datatypes.rst | 2 + docs/source/status.rst | 2 + python/pyarrow/__init__.py | 9 +- python/pyarrow/array.pxi | 23 ++ python/pyarrow/includes/libarrow.pxd | 11 + python/pyarrow/lib.pxd | 4 + python/pyarrow/public-api.pxi | 2 + python/pyarrow/scalar.pxi | 5 + python/pyarrow/tests/test_extension_type.py | 92 ++++++++ python/pyarrow/types.pxi | 104 +++++++++ 16 files changed, 781 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/extension/range.cc b/cpp/src/arrow/extension/range.cc index 466951c961fc..372e5df1b69b 100644 --- a/cpp/src/arrow/extension/range.cc +++ b/cpp/src/arrow/extension/range.cc @@ -200,4 +200,146 @@ std::shared_ptr range(std::shared_ptr value_type, RangeClose return std::move(result).ValueOrDie(); } +// --------------------------------------------------------------------------- +// RangeIncType + +namespace { + +/// Build the storage Struct type for a per-value-inclusivity range. In addition +/// to the "lower"/"upper" bounds (nullable iff unbounded endpoints are allowed), +/// it carries two non-nullable boolean fields recording each bound's inclusivity. +std::shared_ptr MakeIncStorageType(const std::shared_ptr& value_type, + bool allow_unbounded) { + return struct_({field("lower", value_type, allow_unbounded), + field("upper", value_type, allow_unbounded), + field("lower_inc", boolean(), /*nullable=*/false), + field("upper_inc", boolean(), /*nullable=*/false)}); +} + +} // namespace + +std::shared_ptr RangeIncType::value_type() const { + // storage_type() is a struct whose "lower"/"upper" fields share the same type. + return internal::checked_cast(*storage_type()).field(0)->type(); +} + +std::string RangeIncType::ToString(bool show_metadata) const { + std::stringstream ss; + ss << "extension<" << this->extension_name() + << "[value_type=" << value_type()->ToString(show_metadata) << "]>"; + return ss.str(); +} + +bool RangeIncType::ExtensionEquals(const ExtensionType& other) const { + if (extension_name() != other.extension_name()) { + return false; + } + // All parameters (value type, bound nullability, the boolean flag fields) are + // part of the storage type, so a storage comparison is sufficient. + return storage_type()->Equals(*other.storage_type()); +} + +std::string RangeIncType::Serialize() const { + // Inclusivity is stored per value, so there is no type-level parameter to + // serialize. Emit an empty JSON object for explicitness and forward-compat. + return "{}"; +} + +Result> RangeIncType::Deserialize( + std::shared_ptr storage_type, const std::string& serialized_data) const { + // Validate storage type structure. + if (storage_type->id() != Type::STRUCT) { + return Status::Invalid("RangeIncType storage type must be a Struct, got ", + storage_type->ToString()); + } + const auto& struct_type = internal::checked_cast(*storage_type); + if (struct_type.num_fields() != 4) { + return Status::Invalid("RangeIncType storage Struct must have exactly 4 fields, got ", + struct_type.num_fields()); + } + const auto& lower_field = struct_type.field(0); + const auto& upper_field = struct_type.field(1); + const auto& lower_inc_field = struct_type.field(2); + const auto& upper_inc_field = struct_type.field(3); + if (lower_field->name() != "lower") { + return Status::Invalid( + "RangeIncType storage Struct field 0 must be named \"lower\", got \"", + lower_field->name(), "\""); + } + if (upper_field->name() != "upper") { + return Status::Invalid( + "RangeIncType storage Struct field 1 must be named \"upper\", got \"", + upper_field->name(), "\""); + } + if (lower_inc_field->name() != "lower_inc") { + return Status::Invalid( + "RangeIncType storage Struct field 2 must be named \"lower_inc\", got \"", + lower_inc_field->name(), "\""); + } + if (upper_inc_field->name() != "upper_inc") { + return Status::Invalid( + "RangeIncType storage Struct field 3 must be named \"upper_inc\", got \"", + upper_inc_field->name(), "\""); + } + if (!lower_field->type()->Equals(*upper_field->type())) { + return Status::Invalid( + "RangeIncType storage Struct fields \"lower\" and \"upper\" must have the " + "same type, got \"", + lower_field->type()->ToString(), "\" and \"", upper_field->type()->ToString(), + "\""); + } + if (lower_inc_field->type()->id() != Type::BOOL || + upper_inc_field->type()->id() != Type::BOOL) { + return Status::Invalid( + "RangeIncType storage Struct fields \"lower_inc\" and \"upper_inc\" must be " + "boolean, got \"", + lower_inc_field->type()->ToString(), "\" and \"", + upper_inc_field->type()->ToString(), "\""); + } + if (lower_inc_field->nullable() || upper_inc_field->nullable()) { + return Status::Invalid( + "RangeIncType storage Struct fields \"lower_inc\" and \"upper_inc\" must be " + "non-nullable"); + } + + // Unlike RangeType, the metadata carries no parameters: inclusivity lives in + // the storage fields. Accept an empty string or any JSON object (ignoring + // unknown keys for forward compatibility). + if (!serialized_data.empty()) { + rapidjson::Document document; + const auto& parsed = document.Parse(serialized_data.data(), serialized_data.length()); + if (parsed.HasParseError()) { + return Status::Invalid("Invalid serialized JSON data for RangeIncType: ", + rapidjson::GetParseError_En(parsed.GetParseError()), ": ", + serialized_data); + } + if (!document.IsObject()) { + return Status::Invalid( + "Invalid serialized JSON data for RangeIncType: not an object"); + } + } + + return std::make_shared(std::move(storage_type)); +} + +std::shared_ptr RangeIncType::MakeArray(std::shared_ptr data) const { + DCHECK_EQ(data->type->id(), Type::EXTENSION); + DCHECK_EQ("arrow.range_inc", + internal::checked_cast(*data->type).extension_name()); + return std::make_shared(data); +} + +Result> RangeIncType::Make(std::shared_ptr value_type, + bool allow_unbounded) { + auto storage = MakeIncStorageType(value_type, allow_unbounded); + return std::make_shared(std::move(storage)); +} + +std::shared_ptr range_inc(std::shared_ptr value_type, + bool allow_unbounded) { + auto result = RangeIncType::Make(std::move(value_type), allow_unbounded); + ARROW_CHECK_OK(result.status()); + return std::move(result).ValueOrDie(); +} + } // namespace arrow::extension diff --git a/cpp/src/arrow/extension/range.h b/cpp/src/arrow/extension/range.h index b9f5f55e6106..87322e6ec0a7 100644 --- a/cpp/src/arrow/extension/range.h +++ b/cpp/src/arrow/extension/range.h @@ -108,4 +108,77 @@ ARROW_EXPORT std::shared_ptr range(std::shared_ptr value_typ RangeClosed closed = RangeClosed::Left, bool allow_unbounded = true); +/// \brief RangeIncType represents a bounded set (mathematical interval) over an +/// orderable Arrow type T whose bound inclusivity is stored **per value**. +/// +/// Unlike RangeType, which carries a single type-level "closed" parameter, this +/// type records the inclusivity of each row's bounds in two boolean storage +/// fields. This is required for continuous ranges (e.g. PostgreSQL's +/// ``numrange``, ``tsrange``, ``tstzrange``) which cannot be canonicalized to a +/// uniform closedness. It mirrors PostgreSQL's internal range representation. +/// +/// Storage is a Struct with exactly four fields, in order: +/// - "lower": T (null, when nullable = unbounded below, i.e. -infinity) +/// - "upper": T (null, when nullable = unbounded above, i.e. +infinity) +/// - "lower_inc": boolean, non-nullable: is the lower bound inclusive? +/// - "upper_inc": boolean, non-nullable: is the upper bound inclusive? +/// +/// "lower" and "upper" share the same orderable type T and may independently be +/// nullable to represent an unbounded (infinite) endpoint. A null (infinite) +/// bound is always treated as exclusive, regardless of its "*_inc" flag. +/// +/// The outer struct's validity bit marks a null/absent range. +/// +/// There is no type-level "closed" parameter, so the extension metadata carries +/// no parameters (serialized as the empty JSON object ``{}``). +class ARROW_EXPORT RangeIncType : public ExtensionType { + public: + /// \brief Construct a RangeIncType. + /// + /// \param[in] storage_type A four-field Struct type with fields "lower", + /// "upper" (same orderable type T) and non-nullable boolean "lower_inc", + /// "upper_inc". + explicit RangeIncType(std::shared_ptr storage_type) + : ExtensionType(std::move(storage_type)) {} + + std::string extension_name() const override { return "arrow.range_inc"; } + std::string ToString(bool show_metadata = false) const override; + bool ExtensionEquals(const ExtensionType& other) const override; + std::string Serialize() const override; + Result> Deserialize( + std::shared_ptr storage_type, + const std::string& serialized_data) const override; + + /// \brief Create a RangeIncArray from ArrayData. + std::shared_ptr MakeArray(std::shared_ptr data) const override; + + /// \brief Factory function. + /// + /// Constructs the four-field struct storage type internally. + /// \param[in] value_type The orderable Arrow subtype T for lower and upper. + /// \param[in] allow_unbounded Whether each side may be unbounded (infinite). + /// When true, the "lower" and "upper" fields are nullable and a null bound + /// denotes an infinite endpoint; when false, both bounds are non-nullable + /// and the range is always finite. The "lower_inc" and "upper_inc" fields + /// are always non-nullable. Defaults to true. + static Result> Make(std::shared_ptr value_type, + bool allow_unbounded = true); + + /// \brief Return the Arrow subtype T (the type of "lower" and "upper" fields). + std::shared_ptr value_type() const; +}; + +/// \brief Array class for arrow.range_inc extension arrays. +class ARROW_EXPORT RangeIncArray : public ExtensionArray { + public: + using ExtensionArray::ExtensionArray; +}; + +/// \brief Create a RangeIncType with the given value subtype. +/// +/// This is a convenience wrapper around RangeIncType::Make that aborts on error. +/// For recoverable error handling prefer RangeIncType::Make. +ARROW_EXPORT std::shared_ptr range_inc(std::shared_ptr value_type, + bool allow_unbounded = true); + } // namespace arrow::extension diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc index 4b18327e42e1..be529d372be3 100644 --- a/cpp/src/arrow/extension/range_test.cc +++ b/cpp/src/arrow/extension/range_test.cc @@ -317,4 +317,235 @@ TEST(RangeType, BatchRoundTrip) { ASSERT_BATCHES_EQUAL(*batch, *written); } +// =========================================================================== +// RangeIncType -- per-value bound inclusivity +// =========================================================================== + +namespace { + +std::shared_ptr IncStorage(const std::shared_ptr& value_type, + bool nullable_bounds = true) { + return struct_({field("lower", value_type, nullable_bounds), + field("upper", value_type, nullable_bounds), + field("lower_inc", boolean(), /*nullable=*/false), + field("upper_inc", boolean(), /*nullable=*/false)}); +} + +} // namespace + +// --------------------------------------------------------------------------- +// Basics + +TEST(RangeIncType, Basics) { + auto type = + checked_pointer_cast(extension::range_inc(int32())); + ASSERT_EQ("arrow.range_inc", type->extension_name()); + ASSERT_EQ(*int32(), *type->value_type()); + ASSERT_EQ(*type, *type); + ASSERT_NE(*arrow::null(), *type); + // No type-level parameters: metadata is the empty JSON object. + ASSERT_EQ("{}", type->Serialize()); + ASSERT_EQ("extension", type->ToString(false)); + // Storage carries the two non-nullable boolean inclusivity fields. + const auto& storage = internal::checked_cast(*type->storage_type()); + ASSERT_EQ(4, storage.num_fields()); + ASSERT_EQ("lower_inc", storage.field(2)->name()); + ASSERT_EQ("upper_inc", storage.field(3)->name()); + ASSERT_EQ(*boolean(), *storage.field(2)->type()); + ASSERT_FALSE(storage.field(2)->nullable()); + ASSERT_FALSE(storage.field(3)->nullable()); +} + +// --------------------------------------------------------------------------- +// Equals + +TEST(RangeIncType, Equals) { + auto i32 = checked_pointer_cast(extension::range_inc(int32())); + auto i32b = + checked_pointer_cast(extension::range_inc(int32())); + auto i64 = checked_pointer_cast(extension::range_inc(int64())); + auto i32_finite = checked_pointer_cast( + extension::range_inc(int32(), /*allow_unbounded=*/false)); + + // Same object / same parameters. + ASSERT_EQ(*i32, *i32); + ASSERT_EQ(*i32, *i32b); + + // Different value type. + ASSERT_NE(*i32, *i64); + + // Different bound nullability is part of storage, hence a different type. + ASSERT_NE(*i32, *i32_finite); + + // Not equal to non-range types, including a plain arrow.range. + ASSERT_NE(*i32, *arrow::int32()); + ASSERT_NE(*i32, *extension::range(int32())); +} + +// --------------------------------------------------------------------------- +// CreateFromArray + +TEST(RangeIncType, CreateFromArray) { + auto type = + checked_pointer_cast(extension::range_inc(int32())); + auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); + auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); + auto lower_inc = ArrayFromJSON(boolean(), "[true, false, true]"); + auto upper_inc = ArrayFromJSON(boolean(), "[false, false, true]"); + ASSERT_OK_AND_ASSIGN(auto storage, + StructArray::Make({lower, upper, lower_inc, upper_inc}, + type->storage_type()->fields())); + auto array = ExtensionType::WrapArray(type, storage); + ASSERT_EQ(3, array->length()); + ASSERT_EQ(0, array->null_count()); +} + +// --------------------------------------------------------------------------- +// Deserialize - valid cases (metadata carries no parameters) + +TEST(RangeIncType, DeserializeMetadata) { + auto type = + checked_pointer_cast(extension::range_inc(int32())); + + // Empty string, empty object, and extra keys are all accepted. + for (const auto& serialized : + {std::string(""), std::string("{}"), std::string(R"({"extra": 42})")}) { + ASSERT_OK_AND_ASSIGN(auto deserialized, + type->Deserialize(type->storage_type(), serialized)); + ASSERT_EQ(*type, *deserialized) << "Failed for metadata: " << serialized; + } +} + +// --------------------------------------------------------------------------- +// Deserialize - invalid cases + +TEST(RangeIncType, DeserializeInvalidMetadata) { + auto type = + checked_pointer_cast(extension::range_inc(int32())); + + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, + testing::HasSubstr("Missing a name for object member"), + type->Deserialize(type->storage_type(), "{")); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("not an object"), + type->Deserialize(type->storage_type(), "[]")); +} + +TEST(RangeIncType, DeserializeInvalidStorage) { + auto type = + checked_pointer_cast(extension::range_inc(int32())); + + // Not a struct. + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("must be a Struct"), + type->Deserialize(int32(), "{}")); + + // Wrong number of fields (a plain 2-field range struct). + auto two_fields = + struct_({field("lower", int32(), true), field("upper", int32(), true)}); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("exactly 4 fields"), + type->Deserialize(two_fields, "{}")); + + // Wrong inc field names. + auto bad_inc_name = + struct_({field("lower", int32(), true), field("upper", int32(), true), + field("lo_inc", boolean(), false), field("upper_inc", boolean(), false)}); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("named \"lower_inc\""), + type->Deserialize(bad_inc_name, "{}")); + + // Inc fields not boolean. + auto non_bool_inc = + struct_({field("lower", int32(), true), field("upper", int32(), true), + field("lower_inc", int8(), false), field("upper_inc", int8(), false)}); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("must be boolean"), + type->Deserialize(non_bool_inc, "{}")); + + // Inc fields nullable: rejected (would produce ambiguous data). + auto nullable_inc = + struct_({field("lower", int32(), true), field("upper", int32(), true), + field("lower_inc", boolean(), true), field("upper_inc", boolean(), true)}); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("must be non-nullable"), + type->Deserialize(nullable_inc, "{}")); + + // Bounds have different types. + auto mismatched = struct_({field("lower", int32(), true), field("upper", int64(), true), + field("lower_inc", boolean(), false), + field("upper_inc", boolean(), false)}); + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("same type"), + type->Deserialize(mismatched, "{}")); +} + +// --------------------------------------------------------------------------- +// Non-nullable bounds + +TEST(RangeIncType, NonNullableBounds) { + auto type = + checked_pointer_cast(extension::range_inc(int32())); + + // Both bounds non-nullable: accepted (a finite-only range). + ASSERT_OK_AND_ASSIGN( + auto from_non_nullable, + type->Deserialize(IncStorage(int32(), /*nullable_bounds=*/false), "{}")); + ASSERT_EQ( + *int32(), + *checked_pointer_cast(from_non_nullable)->value_type()); + + // The factory can build non-nullable bounds via allow_unbounded=false. + auto finite = checked_pointer_cast( + extension::range_inc(int32(), /*allow_unbounded=*/false)); + const auto& finite_storage = + internal::checked_cast(*finite->storage_type()); + ASSERT_FALSE(finite_storage.field(0)->nullable()); + ASSERT_FALSE(finite_storage.field(1)->nullable()); + // The inc fields are non-nullable regardless of allow_unbounded. + ASSERT_FALSE(finite_storage.field(2)->nullable()); + ASSERT_FALSE(finite_storage.field(3)->nullable()); +} + +// --------------------------------------------------------------------------- +// Metadata round-trip + +TEST(RangeIncType, MetadataRoundTrip) { + for (const auto& type : + {extension::range_inc(int32()), extension::range_inc(int64()), + extension::range_inc(date32()), extension::range_inc(int32(), false)}) { + auto rt = checked_pointer_cast(type); + std::string serialized = rt->Serialize(); + ASSERT_OK_AND_ASSIGN(auto deserialized, + rt->Deserialize(rt->storage_type(), serialized)); + ASSERT_EQ(*type, *deserialized) << "Round-trip failed for: " << type->ToString(); + } +} + +// --------------------------------------------------------------------------- +// IPC (BatchRoundTrip) -- registration round-trip + +TEST(RangeIncType, BatchRoundTrip) { + auto type = + checked_pointer_cast(extension::range_inc(int32())); + auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); + auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); + auto lower_inc = ArrayFromJSON(boolean(), "[true, false, true]"); + auto upper_inc = ArrayFromJSON(boolean(), "[false, false, true]"); + ASSERT_OK_AND_ASSIGN(auto storage, + StructArray::Make({lower, upper, lower_inc, upper_inc}, + type->storage_type()->fields())); + auto array = ExtensionType::WrapArray(type, storage); + auto batch = RecordBatch::Make(schema({field("rng", type)}), array->length(), {array}); + + std::shared_ptr written; + { + ASSERT_OK_AND_ASSIGN(auto out_stream, io::BufferOutputStream::Create()); + ASSERT_OK(ipc::WriteRecordBatchStream({batch}, ipc::IpcWriteOptions::Defaults(), + out_stream.get())); + ASSERT_OK_AND_ASSIGN(auto complete_ipc_stream, out_stream->Finish()); + + io::BufferReader reader(complete_ipc_stream); + std::shared_ptr batch_reader; + ASSERT_OK_AND_ASSIGN(batch_reader, ipc::RecordBatchStreamReader::Open(&reader)); + ASSERT_OK(batch_reader->ReadNext(&written)); + } + + ASSERT_EQ(*batch->schema(), *written->schema()); + ASSERT_BATCHES_EQUAL(*batch, *written); +} + } // namespace arrow diff --git a/cpp/src/arrow/extension_type.cc b/cpp/src/arrow/extension_type.cc index 1dd840621f20..bb673bbc416f 100644 --- a/cpp/src/arrow/extension_type.cc +++ b/cpp/src/arrow/extension_type.cc @@ -158,6 +158,7 @@ static void CreateGlobalRegistry() { ext_types.push_back(extension::fixed_shape_tensor(int64(), {})); ext_types.push_back(extension::opaque(null(), "", "")); ext_types.push_back(extension::range(int32())); + ext_types.push_back(extension::range_inc(int32())); ext_types.push_back(extension::variable_shape_tensor(int64(), 0)); #endif diff --git a/docs/source/cpp/api/extension.rst b/docs/source/cpp/api/extension.rst index 6b1e2e9a8df0..f2c700e74d78 100644 --- a/docs/source/cpp/api/extension.rst +++ b/docs/source/cpp/api/extension.rst @@ -46,6 +46,10 @@ Extension Type classes :project: arrow_cpp :members: +.. doxygenclass:: arrow::extension::RangeIncType + :project: arrow_cpp + :members: + Extension Array classes ======================= @@ -69,9 +73,16 @@ Extension Array classes :project: arrow_cpp :members: +.. doxygenclass:: arrow::extension::RangeIncArray + :project: arrow_cpp + :members: + Extension functions =================== .. doxygenfunction:: arrow::extension::range :project: arrow_cpp +.. doxygenfunction:: arrow::extension::range_inc + :project: arrow_cpp + diff --git a/docs/source/format/CanonicalExtensions.rst b/docs/source/format/CanonicalExtensions.rst index 40ff32db79d8..cc715b978d6e 100644 --- a/docs/source/format/CanonicalExtensions.rst +++ b/docs/source/format/CanonicalExtensions.rst @@ -661,6 +661,79 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. .. _range types: https://www.postgresql.org/docs/current/rangetypes.html +.. _range_inc_extension: + +Range Inc +========= + +Range Inc represents a bounded set (mathematical interval) over an orderable +Arrow type T whose bound inclusivity is recorded **per value** rather than as a +single type-level parameter. It is the companion of the :ref:`Range +` extension type for ranges that cannot be canonicalized to a +uniform closedness. + +.. note:: + + **When to use** ``arrow.range`` **vs.** ``arrow.range_inc``. + Discrete ranges (e.g. PostgreSQL's ``int4range``, ``int8range``, + ``daterange``) canonicalize to a single closedness (left-closed), so they + are best represented by :ref:`arrow.range `, which stores + the closedness once in the type metadata. Continuous ranges (e.g. + PostgreSQL's ``numrange``, ``tsrange``, ``tstzrange``) **cannot** be + canonicalized: two values may share the same endpoints yet differ in + whether those endpoints are included. ``arrow.range_inc`` stores the + inclusivity of each bound alongside the bound itself, mirroring PostgreSQL's + internal range representation, and is the appropriate choice for that case. + +* Extension name: ``arrow.range_inc``. + +* The storage type of the extension is a ``Struct`` with exactly **four fields, + in order**: + + * ``lower``: the lower bound, type **T**, *optionally nullable*. + When the field is nullable, a null value means the range is unbounded below + (negative infinity). + * ``upper``: the upper bound, type **T**, *optionally nullable*. + When the field is nullable, a null value means the range is unbounded above + (positive infinity). + * ``lower_inc``: a **non-nullable** ``boolean`` -- ``true`` when the lower + bound is inclusive for that value, ``false`` when it is exclusive. + * ``upper_inc``: a **non-nullable** ``boolean`` -- ``true`` when the upper + bound is inclusive for that value, ``false`` when it is exclusive. + + **T** (the *subtype* or *value type*) may be any orderable Arrow type: + integer, floating-point, decimal, date, time, or timestamp types. The + ``lower`` and ``upper`` fields share the same type T, read directly from the + storage struct; the subtype is **not** duplicated in the extension metadata. + + Each of ``lower`` and ``upper`` **may** be nullable, independently of the + other, exactly as in :ref:`arrow.range `: nullability is + only needed to represent an unbounded side. A null bound is **always treated + as exclusive**, regardless of its ``lower_inc`` / ``upper_inc`` flag; positive + and negative infinity can never be included. The ``lower_inc`` and + ``upper_inc`` fields are **always non-nullable**. The outer struct's validity + bit marks a null/absent range (a missing range, distinct from an empty range). + +* Extension type parameters: + + This type has **no** type-level parameters. Unlike :ref:`arrow.range + `, inclusivity is not fixed by the type; it is carried per + value in the ``lower_inc`` and ``upper_inc`` fields. + + For a given value, the range contains every x permitted by its finite bounds + and per-value flags: with both flags ``true`` every x such that + ``lower <= x <= upper``, with both flags ``false`` every x such that + ``lower < x < upper``. A value is *empty* when ``lower > upper``, or when + ``lower == upper`` and at least one of ``lower_inc`` / ``upper_inc`` is + ``false``. + +* Description of the serialization: + + Because inclusivity is stored per value, the type carries no parameters and + the extension metadata is an **empty JSON object** ``{}``. For + forward-compatibility, an empty metadata string is also accepted on read, and + any additional keys in the JSON object should be ignored. + Community Extension Types ========================= diff --git a/docs/source/python/api/datatypes.rst b/docs/source/python/api/datatypes.rst index 71ee00557f33..2f6125443dd8 100644 --- a/docs/source/python/api/datatypes.rst +++ b/docs/source/python/api/datatypes.rst @@ -74,6 +74,7 @@ These should be used to create Arrow data types and schemas. opaque bool8 range_ + range_inc uuid json_ field @@ -148,6 +149,7 @@ implemented by PyArrow. UuidType Bool8Type RangeType + RangeIncType .. _api.types.checking: .. currentmodule:: pyarrow.types diff --git a/docs/source/status.rst b/docs/source/status.rst index 01cb40f093d5..a5c229b7b07e 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -135,6 +135,8 @@ Data Types +-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Range | ✓ | | | | | | | | +-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ +| Range Inc | ✓ | | | | | | | | ++-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ Notes: diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 1c4e82181aee..0b6ad2400e52 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -165,7 +165,8 @@ def print_entry(label, value): union, sparse_union, dense_union, dictionary, run_end_encoded, - bool8, fixed_shape_tensor, json_, opaque, range_, uuid, + bool8, fixed_shape_tensor, json_, opaque, range_, range_inc, + uuid, field, type_for_alias, DataType, DictionaryType, StructType, @@ -177,7 +178,7 @@ def print_entry(label, value): Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, BaseExtensionType, ExtensionType, RunEndEncodedType, Bool8Type, FixedShapeTensorType, - JsonType, OpaqueType, RangeType, UuidType, + JsonType, OpaqueType, RangeType, RangeIncType, UuidType, UnknownExtensionType, register_extension_type, unregister_extension_type, DictionaryMemo, @@ -214,7 +215,7 @@ def print_entry(label, value): Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array, StructArray, ExtensionArray, RunEndEncodedArray, Bool8Array, FixedShapeTensorArray, - JsonArray, OpaqueArray, RangeArray, UuidArray, + JsonArray, OpaqueArray, RangeArray, RangeIncArray, UuidArray, scalar, NA, _NULL as NULL, Scalar, NullScalar, BooleanScalar, Int8Scalar, Int16Scalar, Int32Scalar, Int64Scalar, @@ -233,7 +234,7 @@ def print_entry(label, value): MapScalar, StructScalar, UnionScalar, RunEndEncodedScalar, Bool8Scalar, ExtensionScalar, FixedShapeTensorScalar, JsonScalar, OpaqueScalar, - RangeScalar, UuidScalar) + RangeScalar, RangeIncScalar, UuidScalar) # Buffers, allocation from pyarrow.lib import (DeviceAllocationType, Device, MemoryManager, diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 9b2646f04c5b..09ff72d6cb4f 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -5337,6 +5337,29 @@ cdef class RangeArray(ExtensionArray): """ +cdef class RangeIncArray(ExtensionArray): + """ + Concrete class for range_inc extension arrays. + + Examples + -------- + Define the extension type for a range_inc array + + >>> import pyarrow as pa + >>> range_inc_type = pa.range_inc(pa.float64()) + + Create an extension array + + >>> storage = pa.array( + ... [{"lower": 1.0, "upper": 5.0, "lower_inc": True, "upper_inc": False}], + ... range_inc_type.storage_type, + ... ) + >>> arr = pa.ExtensionArray.from_storage(range_inc_type, storage) + >>> isinstance(arr, pa.RangeIncArray) + True + """ + + cdef dict _array_classes = { _Type_NA: NullArray, _Type_BOOL: BooleanArray, diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index aa647f7f607a..9d37703c1d1a 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3155,6 +3155,17 @@ cdef extern from "arrow/extension/range.h" namespace "arrow::extension" nogil: cdef cppclass CRangeArray" arrow::extension::RangeArray"(CExtensionArray): pass + cdef cppclass CRangeIncType" arrow::extension::RangeIncType"(CExtensionType): + + @staticmethod + CResult[shared_ptr[CDataType]] Make(shared_ptr[CDataType] value_type, + c_bool allow_unbounded) + + shared_ptr[CDataType] value_type() + + cdef cppclass CRangeIncArray" arrow::extension::RangeIncArray"(CExtensionArray): + pass + cdef extern from "arrow/util/compression.h" namespace "arrow" nogil: cdef enum CCompressionType" arrow::Compression::type": diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index d1226699ca86..3ca1fb836b14 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -207,6 +207,10 @@ cdef class RangeType(BaseExtensionType): cdef: const CRangeType* range_ext_type +cdef class RangeIncType(BaseExtensionType): + cdef: + const CRangeIncType* range_inc_ext_type + cdef class OpaqueType(BaseExtensionType): cdef: const COpaqueType* opaque_ext_type diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi index 7bb56435b7e2..6655fe6b279c 100644 --- a/python/pyarrow/public-api.pxi +++ b/python/pyarrow/public-api.pxi @@ -133,6 +133,8 @@ cdef api object pyarrow_wrap_data_type( out = FixedShapeTensorType.__new__(FixedShapeTensorType) elif extension_name == b"arrow.range": out = RangeType.__new__(RangeType) + elif extension_name == b"arrow.range_inc": + out = RangeIncType.__new__(RangeIncType) elif extension_name == b"arrow.opaque": out = OpaqueType.__new__(OpaqueType) elif extension_name == b"arrow.uuid": diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi index 86546e2a604d..5a6c2b509309 100644 --- a/python/pyarrow/scalar.pxi +++ b/python/pyarrow/scalar.pxi @@ -1646,6 +1646,11 @@ cdef class RangeScalar(ExtensionScalar): Concrete class for range extension scalar. """ +cdef class RangeIncScalar(ExtensionScalar): + """ + Concrete class for range_inc extension scalar. + """ + cdef dict _scalar_classes = { _Type_BOOL: BooleanScalar, _Type_UINT8: UInt8Scalar, diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py index 41d33ff70152..b8067a9ff34d 100644 --- a/python/pyarrow/tests/test_extension_type.py +++ b/python/pyarrow/tests/test_extension_type.py @@ -2168,6 +2168,98 @@ def test_range_type_allow_unbounded(): assert arr.type == finite +@pytest.mark.parametrize("value_type,rows", [ + (pa.int32(), [ + {"lower": 1, "upper": 5, "lower_inc": True, "upper_inc": False}, + {"lower": None, "upper": 10, "lower_inc": False, "upper_inc": True}, + ]), + (pa.float64(), [ + {"lower": 0.0, "upper": 1.5, "lower_inc": True, "upper_inc": True}, + None, + ]), +]) +def test_range_inc_type(pickle_module, value_type, rows): + range_type = pa.range_inc(value_type) + assert range_type.extension_name == "arrow.range_inc" + assert range_type.value_type == value_type + # Storage carries the two bounds plus per-value, non-nullable inclusivity flags. + assert range_type.storage_type == pa.struct([ + pa.field("lower", value_type, nullable=True), + pa.field("upper", value_type, nullable=True), + pa.field("lower_inc", pa.bool_(), nullable=False), + pa.field("upper_inc", pa.bool_(), nullable=False), + ]) + assert "arrow.range_inc" in str(range_type) + # No type-level closed parameter. + assert not hasattr(range_type, "closed") + + assert range_type == range_type + assert range_type == pa.range_inc(value_type) + assert range_type != value_type + # different value type -> not equal + assert range_type != pa.range_inc(pa.decimal128(12, 3)) + # distinct from a plain arrow.range over the same value type + assert range_type != pa.range_(value_type) + + # Pickle roundtrip + result = pickle_module.loads(pickle_module.dumps(range_type)) + assert result == range_type + assert result.value_type == value_type + + # IPC roundtrip + range_arr_class = range_type.__arrow_ext_class__() + storage = pa.array(rows, range_type.storage_type) + arr = pa.ExtensionArray.from_storage(range_type, storage) + assert isinstance(arr, range_arr_class) + + # extension is registered by default + buf = ipc_write_batch(pa.RecordBatch.from_arrays([arr], ["ext"])) + batch = ipc_read_batch(buf) + + assert batch.column(0).type.extension_name == "arrow.range_inc" + assert batch.column(0).type.value_type == value_type + assert isinstance(batch.column(0), range_arr_class) + assert batch.column(0) == arr + + # cast storage -> extension type + result = storage.cast(range_type) + assert result == arr + + # cast extension type -> storage type + inner = arr.cast(range_type.storage_type) + assert inner == storage + + +def test_range_inc_type_allow_unbounded(): + # Default: bounds are nullable (can represent an unbounded / infinite side). + nullable = pa.range_inc(pa.int32()) + assert nullable.storage_type.field("lower").nullable + assert nullable.storage_type.field("upper").nullable + # The inclusivity flags are always non-nullable. + assert not nullable.storage_type.field("lower_inc").nullable + assert not nullable.storage_type.field("upper_inc").nullable + + # allow_unbounded=False: a finite-only range with non-nullable bounds. + finite = pa.range_inc(pa.int32(), allow_unbounded=False) + assert not finite.storage_type.field("lower").nullable + assert not finite.storage_type.field("upper").nullable + # The flags stay non-nullable regardless of allow_unbounded. + assert not finite.storage_type.field("lower_inc").nullable + assert not finite.storage_type.field("upper_inc").nullable + assert finite.value_type == pa.int32() + + # Distinct types: storage nullability differs. + assert finite != nullable + + # A non-nullable-bounds range_inc round-trips through its storage. + storage = pa.array( + [{"lower": 1, "upper": 5, "lower_inc": True, "upper_inc": False}], + finite.storage_type, + ) + arr = pa.ExtensionArray.from_storage(finite, storage) + assert arr.type == finite + + def test_bool8_type(pickle_module): bool8_type = pa.bool8() storage_type = pa.int8() diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index c9b9faaad2d1..322ed0cdb555 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -2144,6 +2144,50 @@ cdef class RangeType(BaseExtensionType): return RangeScalar +cdef class RangeIncType(BaseExtensionType): + """ + Concrete class for range_inc extension type. + + Like :class:`RangeType`, this represents a bounded set (a mathematical + interval) over an orderable Arrow value type, but the inclusivity of each + bound is stored *per value* rather than as a single type-level parameter. + The underlying storage is a Struct with four fields: "lower" and "upper" + (the value type, each optionally nullable to denote an unbounded endpoint) + and non-nullable boolean "lower_inc" and "upper_inc" recording whether each + bound is inclusive. This is required for continuous ranges (e.g. + PostgreSQL's ``numrange``, ``tsrange``, ``tstzrange``) that cannot be + canonicalized to a uniform closedness. + + Examples + -------- + Create an instance of range_inc extension type: + + >>> import pyarrow as pa + >>> pa.range_inc(pa.float64()) + RangeIncType(extension) + """ + + cdef void init(self, const shared_ptr[CDataType]& type) except *: + BaseExtensionType.init(self, type) + self.range_inc_ext_type = type.get() + + @property + def value_type(self): + """ + The Arrow value type of the "lower" and "upper" bounds. + """ + return pyarrow_wrap_data_type(self.range_inc_ext_type.value_type()) + + def __arrow_ext_class__(self): + return RangeIncArray + + def __reduce__(self): + return range_inc, (self.value_type,) + + def __arrow_ext_scalar_class__(self): + return RangeIncScalar + + cdef class OpaqueType(BaseExtensionType): """ Concrete class for opaque extension type. @@ -5831,6 +5875,66 @@ def range_(DataType value_type not None, str closed="left", allow_unbounded=True return out +def range_inc(DataType value_type not None, allow_unbounded=True): + """ + Create instance of range_inc extension type. + + Unlike :func:`range_`, the inclusivity of each bound is stored per value + (in boolean "lower_inc"/"upper_inc" storage fields) rather than as a single + type-level "closed" parameter, so there is no "closed" argument. This is + required for continuous ranges (e.g. PostgreSQL's ``numrange``, ``tsrange``, + ``tstzrange``) that cannot be canonicalized to a uniform closedness. + + Parameters + ---------- + value_type : DataType + The orderable Arrow type of the "lower" and "upper" interval bounds. + allow_unbounded : bool, default True + Whether each side may be unbounded (infinite). When True the "lower" and + "upper" storage fields are nullable (a null bound is an infinite + endpoint); when False both bounds are non-nullable and the range is + always finite. The "lower_inc" and "upper_inc" fields are always + non-nullable. + + Examples + -------- + Create an instance of a range_inc extension type: + + >>> import pyarrow as pa + >>> type = pa.range_inc(pa.float64()) + >>> type + RangeIncType(extension) + + Inspect the data type: + + >>> type.value_type + DataType(double) + >>> type.storage_type + StructType(struct) + + Create a range_inc array: + + >>> storage = pa.array( + ... [{"lower": 1.0, "upper": 5.0, "lower_inc": True, "upper_inc": False}], + ... type.storage_type, + ... ) + >>> arr = pa.ExtensionArray.from_storage(type, storage) + >>> arr.type + RangeIncType(extension) + + Returns + ------- + type : RangeIncType + """ + + cdef: + shared_ptr[CDataType] c_type = GetResultValue( + CRangeIncType.Make(value_type.sp_type, allow_unbounded)) + RangeIncType out = RangeIncType.__new__(RangeIncType) + out.init(c_type) + return out + + def opaque(DataType storage_type, str type_name not None, str vendor_name not None): """ Create instance of opaque extension type. From b3d071f23d1932d9a3d6ee98e236dbe6a0441ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Thu, 4 Jun 2026 22:44:15 +0000 Subject: [PATCH 11/13] fix(format): rename range_test helper to avoid unity-build clash Under CMAKE_UNITY_BUILD (Windows CI), range_test.cc and opaque_test.cc are merged into one translation unit. Both declared a CheckDeserialize helper (range's in an anonymous namespace, opaque's in namespace arrow), making the unqualified call ambiguous and failing the MSVC build with C2668. Rename the range helper to CheckRangeDeserialize to remove the collision. --- cpp/src/arrow/extension/range_test.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc index be529d372be3..44cbf9958b3b 100644 --- a/cpp/src/arrow/extension/range_test.cc +++ b/cpp/src/arrow/extension/range_test.cc @@ -132,8 +132,8 @@ TEST(RangeType, CreateFromArray) { namespace { -void CheckDeserialize(const std::string& serialized, - const std::shared_ptr& expected) { +void CheckRangeDeserialize(const std::string& serialized, + const std::shared_ptr& expected) { auto type = checked_pointer_cast(expected); ASSERT_OK_AND_ASSIGN(auto deserialized, type->Deserialize(type->storage_type(), serialized)); @@ -145,20 +145,20 @@ void CheckDeserialize(const std::string& serialized, TEST(RangeType, Deserialize) { // Normal JSON ASSERT_NO_FATAL_FAILURE( - CheckDeserialize(R"({"closed": "right"})", - extension::range(int32(), extension::RangeClosed::Right))); - ASSERT_NO_FATAL_FAILURE(CheckDeserialize( + CheckRangeDeserialize(R"({"closed": "right"})", + extension::range(int32(), extension::RangeClosed::Right))); + ASSERT_NO_FATAL_FAILURE(CheckRangeDeserialize( R"({"closed": "left"})", extension::range(int32(), extension::RangeClosed::Left))); - ASSERT_NO_FATAL_FAILURE(CheckDeserialize( + ASSERT_NO_FATAL_FAILURE(CheckRangeDeserialize( R"({"closed": "both"})", extension::range(int32(), extension::RangeClosed::Both))); ASSERT_NO_FATAL_FAILURE( - CheckDeserialize(R"({"closed": "neither"})", - extension::range(int32(), extension::RangeClosed::Neither))); + CheckRangeDeserialize(R"({"closed": "neither"})", + extension::range(int32(), extension::RangeClosed::Neither))); // Extra fields are tolerated (forward-compatibility). ASSERT_NO_FATAL_FAILURE( - CheckDeserialize(R"({"closed": "right", "extra": 42})", - extension::range(int32(), extension::RangeClosed::Right))); + CheckRangeDeserialize(R"({"closed": "right", "extra": 42})", + extension::range(int32(), extension::RangeClosed::Right))); } TEST(RangeType, DefaultClosedIsLeft) { From ad3168a1e34f934c23d5491a1abd7451a46e5882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sat, 12 Sep 2026 15:49:14 +0000 Subject: [PATCH 12/13] refactor(format): migrate arrow.range JSON metadata to simdjson Use JsonWriter for Serialize and the simdjson DOM helpers for Deserialize, matching the other canonical extension types. --- cpp/src/arrow/extension/range.cc | 69 ++++++++++----------------- cpp/src/arrow/extension/range_test.cc | 10 ++-- 2 files changed, 31 insertions(+), 48 deletions(-) diff --git a/cpp/src/arrow/extension/range.cc b/cpp/src/arrow/extension/range.cc index 372e5df1b69b..faf42ebba053 100644 --- a/cpp/src/arrow/extension/range.cc +++ b/cpp/src/arrow/extension/range.cc @@ -20,12 +20,12 @@ #include #include -#include "arrow/json/rapidjson_defs.h" // IWYU pragma: keep #include "arrow/util/logging_internal.h" +#include "arrow/util/simdjson_internal.h" -#include -#include -#include +#include + +using ::arrow::internal::JsonWriter; namespace arrow::extension { @@ -96,19 +96,16 @@ bool RangeType::ExtensionEquals(const ExtensionType& other) const { } std::string RangeType::Serialize() const { - rapidjson::Document document; - document.SetObject(); - rapidjson::Document::AllocatorType& allocator = document.GetAllocator(); - - auto closed_str = ClosedToString(closed_); - rapidjson::Value closed_value( - closed_str.data(), static_cast(closed_str.size()), allocator); - document.AddMember(rapidjson::Value("closed", allocator), closed_value, allocator); - - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - document.Accept(writer); - return buffer.GetString(); + JsonWriter writer; + + writer.StartObject(); + writer.StringField("closed", ClosedToString(closed_)); + writer.EndObject(); + + Result json = writer.GetString(); + // can only fail in OutOfMemory scenarios + ARROW_CHECK_OK(json.status()); + return std::string(*json); } Result> RangeType::Deserialize( @@ -150,28 +147,21 @@ Result> RangeType::Deserialize( "RangeType metadata must be a JSON object with a required \"closed\" key, " "got an empty string"); } - rapidjson::Document document; - const auto& parsed = document.Parse(serialized_data.data(), serialized_data.length()); - if (parsed.HasParseError()) { - return Status::Invalid("Invalid serialized JSON data for RangeType: ", - rapidjson::GetParseError_En(parsed.GetParseError()), ": ", - serialized_data); - } - if (!document.IsObject()) { - return Status::Invalid("Invalid serialized JSON data for RangeType: not an object"); - } - if (!document.HasMember("closed")) { + simdjson::dom::parser parser; + ARROW_ASSIGN_OR_RAISE(auto object, internal::ParseJsonObject(parser, serialized_data)); + + ARROW_ASSIGN_OR_RAISE(auto closed_value, + internal::GetOptionalJsonField(object, "closed")); + if (!closed_value.has_value()) { return Status::Invalid("RangeType metadata is missing the required \"closed\" key: ", serialized_data); } - const auto& closed_val = document["closed"]; - if (!closed_val.IsString()) { + std::string_view closed_str; + if (closed_value->get_string().get(closed_str) != simdjson::SUCCESS) { return Status::Invalid( "Invalid serialized JSON data for RangeType: \"closed\" is not a string"); } - ARROW_ASSIGN_OR_RAISE(RangeClosed closed, - ClosedFromString(std::string_view(closed_val.GetString(), - closed_val.GetStringLength()))); + ARROW_ASSIGN_OR_RAISE(RangeClosed closed, ClosedFromString(closed_str)); return std::make_shared(std::move(storage_type), closed); } @@ -306,17 +296,8 @@ Result> RangeIncType::Deserialize( // the storage fields. Accept an empty string or any JSON object (ignoring // unknown keys for forward compatibility). if (!serialized_data.empty()) { - rapidjson::Document document; - const auto& parsed = document.Parse(serialized_data.data(), serialized_data.length()); - if (parsed.HasParseError()) { - return Status::Invalid("Invalid serialized JSON data for RangeIncType: ", - rapidjson::GetParseError_En(parsed.GetParseError()), ": ", - serialized_data); - } - if (!document.IsObject()) { - return Status::Invalid( - "Invalid serialized JSON data for RangeIncType: not an object"); - } + simdjson::dom::parser parser; + RETURN_NOT_OK(internal::ParseJsonObject(parser, serialized_data).status()); } return std::make_shared(std::move(storage_type)); diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc index 44cbf9958b3b..ddad8fc705d3 100644 --- a/cpp/src/arrow/extension/range_test.cc +++ b/cpp/src/arrow/extension/range_test.cc @@ -186,10 +186,11 @@ TEST(RangeType, DeserializeInvalidMetadata) { // Truly malformed JSON fails. EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, - testing::HasSubstr("Missing a name for object member"), + testing::HasSubstr("Invalid serialized JSON data"), type->Deserialize(type->storage_type(), "{")); - EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("not an object"), + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, + testing::HasSubstr("Invalid serialized JSON data"), type->Deserialize(type->storage_type(), "[]")); EXPECT_RAISES_WITH_MESSAGE_THAT( @@ -424,9 +425,10 @@ TEST(RangeIncType, DeserializeInvalidMetadata) { checked_pointer_cast(extension::range_inc(int32())); EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, - testing::HasSubstr("Missing a name for object member"), + testing::HasSubstr("Invalid serialized JSON data"), type->Deserialize(type->storage_type(), "{")); - EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("not an object"), + EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, + testing::HasSubstr("Invalid serialized JSON data"), type->Deserialize(type->storage_type(), "[]")); } From c414320d8f10278361ae81f36675f0a4f0596d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20R=2E=20H=C3=B6lzlwimmer?= Date: Sun, 13 Sep 2026 00:23:38 +0000 Subject: [PATCH 13/13] refactor(format): rename range types to fixed/variable closedness Name the pair after arrow.fixed_shape_tensor and arrow.variable_shape_tensor: closedness is either one type parameter or stored per value. - arrow.range -> arrow.fixed_closedness_range - arrow.range_inc -> arrow.variable_closedness_range - C++: RangeType/RangeArray -> FixedClosednessRangeType/Array, RangeIncType/RangeIncArray -> VariableClosednessRangeType/Array, range()/range_inc() -> fixed_closedness_range()/variable_closedness_range() - pyarrow: range_/range_inc and the Range* classes follow the same names --- cpp/src/arrow/extension/range.cc | 146 ++++++++------ cpp/src/arrow/extension/range.h | 72 +++---- cpp/src/arrow/extension/range_test.cc | 211 +++++++++++--------- cpp/src/arrow/extension_type.cc | 4 +- docs/source/cpp/api/extension.rst | 12 +- docs/source/format/CanonicalExtensions.rst | 78 ++++---- docs/source/python/api/arrays.rst | 6 +- docs/source/python/api/datatypes.rst | 8 +- docs/source/status.rst | 6 +- python/pyarrow/__init__.py | 19 +- python/pyarrow/array.pxi | 24 +-- python/pyarrow/includes/libarrow.pxd | 12 +- python/pyarrow/lib.pxd | 8 +- python/pyarrow/public-api.pxi | 8 +- python/pyarrow/scalar.pxi | 8 +- python/pyarrow/tests/test_extension_type.py | 61 +++--- python/pyarrow/types.pxi | 114 ++++++----- 17 files changed, 429 insertions(+), 368 deletions(-) diff --git a/cpp/src/arrow/extension/range.cc b/cpp/src/arrow/extension/range.cc index faf42ebba053..f9f0d764e7f6 100644 --- a/cpp/src/arrow/extension/range.cc +++ b/cpp/src/arrow/extension/range.cc @@ -55,13 +55,13 @@ Result ClosedFromString(std::string_view s) { if (s == "both") return RangeClosed::Both; if (s == "neither") return RangeClosed::Neither; return Status::Invalid( - "Invalid value for RangeType \"closed\" parameter: \"", s, + "Invalid value for FixedClosednessRangeType \"closed\" parameter: \"", s, "\". Expected one of: \"left\", \"right\", \"both\", \"neither\"."); } /// Build the storage Struct type for a given value subtype. -std::shared_ptr MakeStorageType(const std::shared_ptr& value_type, - bool allow_unbounded) { +std::shared_ptr MakeFixedClosednessStorageType( + const std::shared_ptr& value_type, bool allow_unbounded) { // Nullable bounds can represent an unbounded (infinite) endpoint; non-nullable // bounds are always finite. return struct_({field("lower", value_type, allow_unbounded), @@ -71,14 +71,14 @@ std::shared_ptr MakeStorageType(const std::shared_ptr& value } // namespace // --------------------------------------------------------------------------- -// RangeType +// FixedClosednessRangeType -std::shared_ptr RangeType::value_type() const { +std::shared_ptr FixedClosednessRangeType::value_type() const { // storage_type() is a struct with two fields; both share the same type. return internal::checked_cast(*storage_type()).field(0)->type(); } -std::string RangeType::ToString(bool show_metadata) const { +std::string FixedClosednessRangeType::ToString(bool show_metadata) const { std::stringstream ss; ss << "extension<" << this->extension_name() << "[value_type=" << value_type()->ToString(show_metadata) @@ -86,16 +86,17 @@ std::string RangeType::ToString(bool show_metadata) const { return ss.str(); } -bool RangeType::ExtensionEquals(const ExtensionType& other) const { +bool FixedClosednessRangeType::ExtensionEquals(const ExtensionType& other) const { if (extension_name() != other.extension_name()) { return false; } - const auto& other_range = internal::checked_cast(other); + const auto& other_range = + internal::checked_cast(other); return storage_type()->Equals(*other_range.storage_type()) && closed_ == other_range.closed_; } -std::string RangeType::Serialize() const { +std::string FixedClosednessRangeType::Serialize() const { JsonWriter writer; writer.StartObject(); @@ -108,33 +109,35 @@ std::string RangeType::Serialize() const { return std::string(*json); } -Result> RangeType::Deserialize( +Result> FixedClosednessRangeType::Deserialize( std::shared_ptr storage_type, const std::string& serialized_data) const { // Validate storage type structure. if (storage_type->id() != Type::STRUCT) { - return Status::Invalid("RangeType storage type must be a Struct, got ", + return Status::Invalid("FixedClosednessRangeType storage type must be a Struct, got ", storage_type->ToString()); } const auto& struct_type = internal::checked_cast(*storage_type); if (struct_type.num_fields() != 2) { - return Status::Invalid("RangeType storage Struct must have exactly 2 fields, got ", - struct_type.num_fields()); + return Status::Invalid( + "FixedClosednessRangeType storage Struct must have exactly 2 fields, got ", + struct_type.num_fields()); } const auto& lower_field = struct_type.field(0); const auto& upper_field = struct_type.field(1); if (lower_field->name() != "lower") { return Status::Invalid( - "RangeType storage Struct field 0 must be named \"lower\", got \"", + "FixedClosednessRangeType storage Struct field 0 must be named \"lower\", got \"", lower_field->name(), "\""); } if (upper_field->name() != "upper") { return Status::Invalid( - "RangeType storage Struct field 1 must be named \"upper\", got \"", + "FixedClosednessRangeType storage Struct field 1 must be named \"upper\", got \"", upper_field->name(), "\""); } if (!lower_field->type()->Equals(*upper_field->type())) { return Status::Invalid( - "RangeType storage Struct fields \"lower\" and \"upper\" must have the same " + "FixedClosednessRangeType storage Struct fields \"lower\" and \"upper\" must " + "have the same " "type, got \"", lower_field->type()->ToString(), "\" and \"", upper_field->type()->ToString(), "\""); @@ -144,7 +147,8 @@ Result> RangeType::Deserialize( // is not defaulted on the wire: empty metadata or a missing key is invalid. if (serialized_data.empty()) { return Status::Invalid( - "RangeType metadata must be a JSON object with a required \"closed\" key, " + "FixedClosednessRangeType metadata must be a JSON object with a required " + "\"closed\" key, " "got an empty string"); } simdjson::dom::parser parser; @@ -153,53 +157,57 @@ Result> RangeType::Deserialize( ARROW_ASSIGN_OR_RAISE(auto closed_value, internal::GetOptionalJsonField(object, "closed")); if (!closed_value.has_value()) { - return Status::Invalid("RangeType metadata is missing the required \"closed\" key: ", - serialized_data); + return Status::Invalid( + "FixedClosednessRangeType metadata is missing the required \"closed\" key: ", + serialized_data); } std::string_view closed_str; if (closed_value->get_string().get(closed_str) != simdjson::SUCCESS) { return Status::Invalid( - "Invalid serialized JSON data for RangeType: \"closed\" is not a string"); + "Invalid serialized JSON data for FixedClosednessRangeType: \"closed\" is not a " + "string"); } ARROW_ASSIGN_OR_RAISE(RangeClosed closed, ClosedFromString(closed_str)); - return std::make_shared(std::move(storage_type), closed); + return std::make_shared(std::move(storage_type), closed); } -std::shared_ptr RangeType::MakeArray(std::shared_ptr data) const { +std::shared_ptr FixedClosednessRangeType::MakeArray( + std::shared_ptr data) const { DCHECK_EQ(data->type->id(), Type::EXTENSION); - DCHECK_EQ("arrow.range", + DCHECK_EQ("arrow.fixed_closedness_range", internal::checked_cast(*data->type).extension_name()); - return std::make_shared(data); + return std::make_shared(data); } -Result> RangeType::Make(std::shared_ptr value_type, - RangeClosed closed, - bool allow_unbounded) { - auto storage = MakeStorageType(value_type, allow_unbounded); - return std::make_shared(std::move(storage), closed); +Result> FixedClosednessRangeType::Make( + std::shared_ptr value_type, RangeClosed closed, bool allow_unbounded) { + auto storage = MakeFixedClosednessStorageType(value_type, allow_unbounded); + return std::make_shared(std::move(storage), closed); } // --------------------------------------------------------------------------- // Free factory function -std::shared_ptr range(std::shared_ptr value_type, RangeClosed closed, - bool allow_unbounded) { - auto result = RangeType::Make(std::move(value_type), closed, allow_unbounded); +std::shared_ptr fixed_closedness_range(std::shared_ptr value_type, + RangeClosed closed, + bool allow_unbounded) { + auto result = + FixedClosednessRangeType::Make(std::move(value_type), closed, allow_unbounded); ARROW_CHECK_OK(result.status()); return std::move(result).ValueOrDie(); } // --------------------------------------------------------------------------- -// RangeIncType +// VariableClosednessRangeType namespace { /// Build the storage Struct type for a per-value-inclusivity range. In addition /// to the "lower"/"upper" bounds (nullable iff unbounded endpoints are allowed), /// it carries two non-nullable boolean fields recording each bound's inclusivity. -std::shared_ptr MakeIncStorageType(const std::shared_ptr& value_type, - bool allow_unbounded) { +std::shared_ptr MakeVariableClosednessStorageType( + const std::shared_ptr& value_type, bool allow_unbounded) { return struct_({field("lower", value_type, allow_unbounded), field("upper", value_type, allow_unbounded), field("lower_inc", boolean(), /*nullable=*/false), @@ -208,19 +216,19 @@ std::shared_ptr MakeIncStorageType(const std::shared_ptr& va } // namespace -std::shared_ptr RangeIncType::value_type() const { +std::shared_ptr VariableClosednessRangeType::value_type() const { // storage_type() is a struct whose "lower"/"upper" fields share the same type. return internal::checked_cast(*storage_type()).field(0)->type(); } -std::string RangeIncType::ToString(bool show_metadata) const { +std::string VariableClosednessRangeType::ToString(bool show_metadata) const { std::stringstream ss; ss << "extension<" << this->extension_name() << "[value_type=" << value_type()->ToString(show_metadata) << "]>"; return ss.str(); } -bool RangeIncType::ExtensionEquals(const ExtensionType& other) const { +bool VariableClosednessRangeType::ExtensionEquals(const ExtensionType& other) const { if (extension_name() != other.extension_name()) { return false; } @@ -229,23 +237,25 @@ bool RangeIncType::ExtensionEquals(const ExtensionType& other) const { return storage_type()->Equals(*other.storage_type()); } -std::string RangeIncType::Serialize() const { +std::string VariableClosednessRangeType::Serialize() const { // Inclusivity is stored per value, so there is no type-level parameter to // serialize. Emit an empty JSON object for explicitness and forward-compat. return "{}"; } -Result> RangeIncType::Deserialize( +Result> VariableClosednessRangeType::Deserialize( std::shared_ptr storage_type, const std::string& serialized_data) const { // Validate storage type structure. if (storage_type->id() != Type::STRUCT) { - return Status::Invalid("RangeIncType storage type must be a Struct, got ", - storage_type->ToString()); + return Status::Invalid( + "VariableClosednessRangeType storage type must be a Struct, got ", + storage_type->ToString()); } const auto& struct_type = internal::checked_cast(*storage_type); if (struct_type.num_fields() != 4) { - return Status::Invalid("RangeIncType storage Struct must have exactly 4 fields, got ", - struct_type.num_fields()); + return Status::Invalid( + "VariableClosednessRangeType storage Struct must have exactly 4 fields, got ", + struct_type.num_fields()); } const auto& lower_field = struct_type.field(0); const auto& upper_field = struct_type.field(1); @@ -253,27 +263,32 @@ Result> RangeIncType::Deserialize( const auto& upper_inc_field = struct_type.field(3); if (lower_field->name() != "lower") { return Status::Invalid( - "RangeIncType storage Struct field 0 must be named \"lower\", got \"", + "VariableClosednessRangeType storage Struct field 0 must be named \"lower\", got " + "\"", lower_field->name(), "\""); } if (upper_field->name() != "upper") { return Status::Invalid( - "RangeIncType storage Struct field 1 must be named \"upper\", got \"", + "VariableClosednessRangeType storage Struct field 1 must be named \"upper\", got " + "\"", upper_field->name(), "\""); } if (lower_inc_field->name() != "lower_inc") { return Status::Invalid( - "RangeIncType storage Struct field 2 must be named \"lower_inc\", got \"", + "VariableClosednessRangeType storage Struct field 2 must be named \"lower_inc\", " + "got \"", lower_inc_field->name(), "\""); } if (upper_inc_field->name() != "upper_inc") { return Status::Invalid( - "RangeIncType storage Struct field 3 must be named \"upper_inc\", got \"", + "VariableClosednessRangeType storage Struct field 3 must be named \"upper_inc\", " + "got \"", upper_inc_field->name(), "\""); } if (!lower_field->type()->Equals(*upper_field->type())) { return Status::Invalid( - "RangeIncType storage Struct fields \"lower\" and \"upper\" must have the " + "VariableClosednessRangeType storage Struct fields \"lower\" and \"upper\" must " + "have the " "same type, got \"", lower_field->type()->ToString(), "\" and \"", upper_field->type()->ToString(), "\""); @@ -281,44 +296,47 @@ Result> RangeIncType::Deserialize( if (lower_inc_field->type()->id() != Type::BOOL || upper_inc_field->type()->id() != Type::BOOL) { return Status::Invalid( - "RangeIncType storage Struct fields \"lower_inc\" and \"upper_inc\" must be " + "VariableClosednessRangeType storage Struct fields \"lower_inc\" and " + "\"upper_inc\" must be " "boolean, got \"", lower_inc_field->type()->ToString(), "\" and \"", upper_inc_field->type()->ToString(), "\""); } if (lower_inc_field->nullable() || upper_inc_field->nullable()) { return Status::Invalid( - "RangeIncType storage Struct fields \"lower_inc\" and \"upper_inc\" must be " + "VariableClosednessRangeType storage Struct fields \"lower_inc\" and " + "\"upper_inc\" must be " "non-nullable"); } - // Unlike RangeType, the metadata carries no parameters: inclusivity lives in - // the storage fields. Accept an empty string or any JSON object (ignoring + // Unlike FixedClosednessRangeType, the metadata carries no parameters: inclusivity + // lives in the storage fields. Accept an empty string or any JSON object (ignoring // unknown keys for forward compatibility). if (!serialized_data.empty()) { simdjson::dom::parser parser; RETURN_NOT_OK(internal::ParseJsonObject(parser, serialized_data).status()); } - return std::make_shared(std::move(storage_type)); + return std::make_shared(std::move(storage_type)); } -std::shared_ptr RangeIncType::MakeArray(std::shared_ptr data) const { +std::shared_ptr VariableClosednessRangeType::MakeArray( + std::shared_ptr data) const { DCHECK_EQ(data->type->id(), Type::EXTENSION); - DCHECK_EQ("arrow.range_inc", + DCHECK_EQ("arrow.variable_closedness_range", internal::checked_cast(*data->type).extension_name()); - return std::make_shared(data); + return std::make_shared(data); } -Result> RangeIncType::Make(std::shared_ptr value_type, - bool allow_unbounded) { - auto storage = MakeIncStorageType(value_type, allow_unbounded); - return std::make_shared(std::move(storage)); +Result> VariableClosednessRangeType::Make( + std::shared_ptr value_type, bool allow_unbounded) { + auto storage = MakeVariableClosednessStorageType(value_type, allow_unbounded); + return std::make_shared(std::move(storage)); } -std::shared_ptr range_inc(std::shared_ptr value_type, - bool allow_unbounded) { - auto result = RangeIncType::Make(std::move(value_type), allow_unbounded); +std::shared_ptr variable_closedness_range(std::shared_ptr value_type, + bool allow_unbounded) { + auto result = VariableClosednessRangeType::Make(std::move(value_type), allow_unbounded); ARROW_CHECK_OK(result.status()); return std::move(result).ValueOrDie(); } diff --git a/cpp/src/arrow/extension/range.h b/cpp/src/arrow/extension/range.h index 87322e6ec0a7..d16488fe3f23 100644 --- a/cpp/src/arrow/extension/range.h +++ b/cpp/src/arrow/extension/range.h @@ -22,7 +22,7 @@ namespace arrow::extension { -/// \brief Which bound(s) of an arrow.range interval are inclusive. +/// \brief Which bound(s) of an arrow.fixed_closedness_range interval are inclusive. /// /// Null (infinite) bounds are always exclusive regardless of this value. enum class RangeClosed { @@ -36,8 +36,8 @@ enum class RangeClosed { Neither, }; -/// \brief RangeType represents a bounded set (mathematical interval) over an -/// orderable Arrow type T. +/// \brief FixedClosednessRangeType represents a bounded set (mathematical interval) over +/// an orderable Arrow type T. /// /// Storage is a Struct with exactly two fields "lower" and "upper" of the same /// orderable type T. Each field may independently be nullable or not: a nullable @@ -50,17 +50,18 @@ enum class RangeClosed { /// /// The "closed" parameter controls which finite bounds are inclusive. /// Null (infinite) bounds are always treated as exclusive. -class ARROW_EXPORT RangeType : public ExtensionType { +class ARROW_EXPORT FixedClosednessRangeType : public ExtensionType { public: - /// \brief Construct a RangeType. + /// \brief Construct a FixedClosednessRangeType. /// /// \param[in] storage_type A two-field Struct type with nullable fields /// "lower" and "upper" of the same orderable Arrow type T. /// \param[in] closed Which bound(s) are inclusive. - explicit RangeType(std::shared_ptr storage_type, RangeClosed closed) + explicit FixedClosednessRangeType(std::shared_ptr storage_type, + RangeClosed closed) : ExtensionType(std::move(storage_type)), closed_(closed) {} - std::string extension_name() const override { return "arrow.range"; } + std::string extension_name() const override { return "arrow.fixed_closedness_range"; } std::string ToString(bool show_metadata = false) const override; bool ExtensionEquals(const ExtensionType& other) const override; std::string Serialize() const override; @@ -68,7 +69,7 @@ class ARROW_EXPORT RangeType : public ExtensionType { std::shared_ptr storage_type, const std::string& serialized_data) const override; - /// \brief Create a RangeArray from ArrayData. + /// \brief Create a FixedClosednessRangeArray from ArrayData. std::shared_ptr MakeArray(std::shared_ptr data) const override; /// \brief Factory function. @@ -94,26 +95,27 @@ class ARROW_EXPORT RangeType : public ExtensionType { RangeClosed closed_; }; -/// \brief Array class for arrow.range extension arrays. -class ARROW_EXPORT RangeArray : public ExtensionArray { +/// \brief Array class for arrow.fixed_closedness_range extension arrays. +class ARROW_EXPORT FixedClosednessRangeArray : public ExtensionArray { public: using ExtensionArray::ExtensionArray; }; -/// \brief Create a RangeType with the given value subtype and closed parameter. +/// \brief Create a FixedClosednessRangeType with the given value subtype and closed +/// parameter. /// -/// This is a convenience wrapper around RangeType::Make that aborts on error. -/// For recoverable error handling prefer RangeType::Make. -ARROW_EXPORT std::shared_ptr range(std::shared_ptr value_type, - RangeClosed closed = RangeClosed::Left, - bool allow_unbounded = true); - -/// \brief RangeIncType represents a bounded set (mathematical interval) over an -/// orderable Arrow type T whose bound inclusivity is stored **per value**. +/// This is a convenience wrapper around FixedClosednessRangeType::Make that aborts on +/// error. For recoverable error handling prefer FixedClosednessRangeType::Make. +ARROW_EXPORT std::shared_ptr fixed_closedness_range( + std::shared_ptr value_type, RangeClosed closed = RangeClosed::Left, + bool allow_unbounded = true); + +/// \brief VariableClosednessRangeType represents a bounded set (mathematical interval) +/// over an orderable Arrow type T whose bound inclusivity is stored **per value**. /// -/// Unlike RangeType, which carries a single type-level "closed" parameter, this -/// type records the inclusivity of each row's bounds in two boolean storage -/// fields. This is required for continuous ranges (e.g. PostgreSQL's +/// Unlike FixedClosednessRangeType, which carries a single type-level "closed" parameter, +/// this type records the inclusivity of each row's bounds in two boolean storage fields. +/// This is required for continuous ranges (e.g. PostgreSQL's /// ``numrange``, ``tsrange``, ``tstzrange``) which cannot be canonicalized to a /// uniform closedness. It mirrors PostgreSQL's internal range representation. /// @@ -131,17 +133,19 @@ ARROW_EXPORT std::shared_ptr range(std::shared_ptr value_typ /// /// There is no type-level "closed" parameter, so the extension metadata carries /// no parameters (serialized as the empty JSON object ``{}``). -class ARROW_EXPORT RangeIncType : public ExtensionType { +class ARROW_EXPORT VariableClosednessRangeType : public ExtensionType { public: - /// \brief Construct a RangeIncType. + /// \brief Construct a VariableClosednessRangeType. /// /// \param[in] storage_type A four-field Struct type with fields "lower", /// "upper" (same orderable type T) and non-nullable boolean "lower_inc", /// "upper_inc". - explicit RangeIncType(std::shared_ptr storage_type) + explicit VariableClosednessRangeType(std::shared_ptr storage_type) : ExtensionType(std::move(storage_type)) {} - std::string extension_name() const override { return "arrow.range_inc"; } + std::string extension_name() const override { + return "arrow.variable_closedness_range"; + } std::string ToString(bool show_metadata = false) const override; bool ExtensionEquals(const ExtensionType& other) const override; std::string Serialize() const override; @@ -149,7 +153,7 @@ class ARROW_EXPORT RangeIncType : public ExtensionType { std::shared_ptr storage_type, const std::string& serialized_data) const override; - /// \brief Create a RangeIncArray from ArrayData. + /// \brief Create a VariableClosednessRangeArray from ArrayData. std::shared_ptr MakeArray(std::shared_ptr data) const override; /// \brief Factory function. @@ -168,17 +172,17 @@ class ARROW_EXPORT RangeIncType : public ExtensionType { std::shared_ptr value_type() const; }; -/// \brief Array class for arrow.range_inc extension arrays. -class ARROW_EXPORT RangeIncArray : public ExtensionArray { +/// \brief Array class for arrow.variable_closedness_range extension arrays. +class ARROW_EXPORT VariableClosednessRangeArray : public ExtensionArray { public: using ExtensionArray::ExtensionArray; }; -/// \brief Create a RangeIncType with the given value subtype. +/// \brief Create a VariableClosednessRangeType with the given value subtype. /// -/// This is a convenience wrapper around RangeIncType::Make that aborts on error. -/// For recoverable error handling prefer RangeIncType::Make. -ARROW_EXPORT std::shared_ptr range_inc(std::shared_ptr value_type, - bool allow_unbounded = true); +/// This is a convenience wrapper around VariableClosednessRangeType::Make that aborts on +/// error. For recoverable error handling prefer VariableClosednessRangeType::Make. +ARROW_EXPORT std::shared_ptr variable_closedness_range( + std::shared_ptr value_type, bool allow_unbounded = true); } // namespace arrow::extension diff --git a/cpp/src/arrow/extension/range_test.cc b/cpp/src/arrow/extension/range_test.cc index ddad8fc705d3..77331dfaad40 100644 --- a/cpp/src/arrow/extension/range_test.cc +++ b/cpp/src/arrow/extension/range_test.cc @@ -36,47 +36,47 @@ using internal::checked_pointer_cast; // --------------------------------------------------------------------------- // Helpers -static std::shared_ptr RangeInt32Right() { - return checked_pointer_cast( - extension::range(int32(), extension::RangeClosed::Right)); +static std::shared_ptr RangeInt32Right() { + return checked_pointer_cast( + extension::fixed_closedness_range(int32(), extension::RangeClosed::Right)); } -static std::shared_ptr RangeInt32Both() { - return checked_pointer_cast( - extension::range(int32(), extension::RangeClosed::Both)); +static std::shared_ptr RangeInt32Both() { + return checked_pointer_cast( + extension::fixed_closedness_range(int32(), extension::RangeClosed::Both)); } -static std::shared_ptr RangeInt64Left() { - return checked_pointer_cast( - extension::range(int64(), extension::RangeClosed::Left)); +static std::shared_ptr RangeInt64Left() { + return checked_pointer_cast( + extension::fixed_closedness_range(int64(), extension::RangeClosed::Left)); } // --------------------------------------------------------------------------- // Basics -TEST(RangeType, Basics) { +TEST(FixedClosednessRangeType, Basics) { auto type = RangeInt32Right(); - ASSERT_EQ("arrow.range", type->extension_name()); + ASSERT_EQ("arrow.fixed_closedness_range", type->extension_name()); ASSERT_EQ(*int32(), *type->value_type()); ASSERT_EQ(extension::RangeClosed::Right, type->closed()); ASSERT_EQ(*type, *type); ASSERT_NE(*arrow::null(), *type); ASSERT_THAT(type->Serialize(), ::testing::Not(::testing::IsEmpty())); ASSERT_EQ(R"({"closed":"right"})", type->Serialize()); - ASSERT_EQ("extension", + ASSERT_EQ("extension", type->ToString(false)); } -TEST(RangeType, AllClosedValues) { +TEST(FixedClosednessRangeType, AllClosedValues) { using C = extension::RangeClosed; - auto left = - checked_pointer_cast(extension::range(int32(), C::Left)); - auto right = - checked_pointer_cast(extension::range(int32(), C::Right)); - auto both = - checked_pointer_cast(extension::range(int32(), C::Both)); - auto neither = - checked_pointer_cast(extension::range(int32(), C::Neither)); + auto left = checked_pointer_cast( + extension::fixed_closedness_range(int32(), C::Left)); + auto right = checked_pointer_cast( + extension::fixed_closedness_range(int32(), C::Right)); + auto both = checked_pointer_cast( + extension::fixed_closedness_range(int32(), C::Both)); + auto neither = checked_pointer_cast( + extension::fixed_closedness_range(int32(), C::Neither)); ASSERT_EQ(R"({"closed":"left"})", left->Serialize()); ASSERT_EQ(R"({"closed":"right"})", right->Serialize()); @@ -87,7 +87,7 @@ TEST(RangeType, AllClosedValues) { // --------------------------------------------------------------------------- // Equals -TEST(RangeType, Equals) { +TEST(FixedClosednessRangeType, Equals) { auto type_i32_right = RangeInt32Right(); auto type_i32_both = RangeInt32Both(); auto type_i64_left = RangeInt64Left(); @@ -113,7 +113,7 @@ TEST(RangeType, Equals) { // --------------------------------------------------------------------------- // CreateFromArray -TEST(RangeType, CreateFromArray) { +TEST(FixedClosednessRangeType, CreateFromArray) { auto type = RangeInt32Right(); // Build a StructArray that matches the storage type. auto storage_type = type->storage_type(); @@ -134,7 +134,7 @@ namespace { void CheckRangeDeserialize(const std::string& serialized, const std::shared_ptr& expected) { - auto type = checked_pointer_cast(expected); + auto type = checked_pointer_cast(expected); ASSERT_OK_AND_ASSIGN(auto deserialized, type->Deserialize(type->storage_type(), serialized)); ASSERT_EQ(*expected, *deserialized); @@ -142,29 +142,32 @@ void CheckRangeDeserialize(const std::string& serialized, } // namespace -TEST(RangeType, Deserialize) { +TEST(FixedClosednessRangeType, Deserialize) { // Normal JSON - ASSERT_NO_FATAL_FAILURE( - CheckRangeDeserialize(R"({"closed": "right"})", - extension::range(int32(), extension::RangeClosed::Right))); ASSERT_NO_FATAL_FAILURE(CheckRangeDeserialize( - R"({"closed": "left"})", extension::range(int32(), extension::RangeClosed::Left))); + R"({"closed": "right"})", + extension::fixed_closedness_range(int32(), extension::RangeClosed::Right))); ASSERT_NO_FATAL_FAILURE(CheckRangeDeserialize( - R"({"closed": "both"})", extension::range(int32(), extension::RangeClosed::Both))); - ASSERT_NO_FATAL_FAILURE( - CheckRangeDeserialize(R"({"closed": "neither"})", - extension::range(int32(), extension::RangeClosed::Neither))); + R"({"closed": "left"})", + extension::fixed_closedness_range(int32(), extension::RangeClosed::Left))); + ASSERT_NO_FATAL_FAILURE(CheckRangeDeserialize( + R"({"closed": "both"})", + extension::fixed_closedness_range(int32(), extension::RangeClosed::Both))); + ASSERT_NO_FATAL_FAILURE(CheckRangeDeserialize( + R"({"closed": "neither"})", + extension::fixed_closedness_range(int32(), extension::RangeClosed::Neither))); // Extra fields are tolerated (forward-compatibility). - ASSERT_NO_FATAL_FAILURE( - CheckRangeDeserialize(R"({"closed": "right", "extra": 42})", - extension::range(int32(), extension::RangeClosed::Right))); + ASSERT_NO_FATAL_FAILURE(CheckRangeDeserialize( + R"({"closed": "right", "extra": 42})", + extension::fixed_closedness_range(int32(), extension::RangeClosed::Right))); } -TEST(RangeType, DefaultClosedIsLeft) { +TEST(FixedClosednessRangeType, DefaultClosedIsLeft) { // The C++ convenience default is left-closed; the wire format still always // carries an explicit "closed". - auto type = checked_pointer_cast(extension::range(int32())); + auto type = checked_pointer_cast( + extension::fixed_closedness_range(int32())); ASSERT_EQ(extension::RangeClosed::Left, type->closed()); ASSERT_EQ(R"({"closed":"left"})", type->Serialize()); } @@ -172,7 +175,7 @@ TEST(RangeType, DefaultClosedIsLeft) { // --------------------------------------------------------------------------- // Deserialize - invalid cases -TEST(RangeType, DeserializeInvalidMetadata) { +TEST(FixedClosednessRangeType, DeserializeInvalidMetadata) { auto type = RangeInt32Right(); // "closed" is required on the wire: empty metadata is invalid. @@ -198,11 +201,11 @@ TEST(RangeType, DeserializeInvalidMetadata) { type->Deserialize(type->storage_type(), R"({"closed": 42})")); EXPECT_RAISES_WITH_MESSAGE_THAT( - Invalid, testing::HasSubstr("Invalid value for RangeType"), + Invalid, testing::HasSubstr("Invalid value for FixedClosednessRangeType"), type->Deserialize(type->storage_type(), R"({"closed": "unknown"})")); } -TEST(RangeType, DeserializeInvalidStorage) { +TEST(FixedClosednessRangeType, DeserializeInvalidStorage) { auto type = RangeInt32Right(); auto wrong_storage_not_struct = int32(); @@ -243,7 +246,7 @@ TEST(RangeType, DeserializeInvalidStorage) { // Bound nullability is only needed to represent an unbounded (infinite) // endpoint; non-nullable bounds describe a finite-only range and are accepted. -TEST(RangeType, NonNullableBounds) { +TEST(FixedClosednessRangeType, NonNullableBounds) { auto type = RangeInt32Right(); // Both bounds non-nullable: accepted (a finite-only range). @@ -252,7 +255,8 @@ TEST(RangeType, NonNullableBounds) { ASSERT_OK_AND_ASSIGN(auto from_non_nullable, type->Deserialize(both_non_nullable, R"({"closed":"right"})")); ASSERT_EQ(*int32(), - *checked_pointer_cast(from_non_nullable)->value_type()); + *checked_pointer_cast(from_non_nullable) + ->value_type()); // Asymmetric: lower nullable (may be -inf), upper non-nullable (always finite). auto asymmetric = struct_({field("lower", int32(), /*nullable=*/true), @@ -260,11 +264,13 @@ TEST(RangeType, NonNullableBounds) { ASSERT_OK_AND_ASSIGN(auto from_asymmetric, type->Deserialize(asymmetric, R"({"closed":"left"})")); ASSERT_EQ(extension::RangeClosed::Left, - checked_pointer_cast(from_asymmetric)->closed()); + checked_pointer_cast(from_asymmetric) + ->closed()); // The factory can build non-nullable bounds via allow_unbounded=false. - auto finite = checked_pointer_cast( - extension::range(int32(), extension::RangeClosed::Both, /*allow_unbounded=*/false)); + auto finite = checked_pointer_cast( + extension::fixed_closedness_range(int32(), extension::RangeClosed::Both, + /*allow_unbounded=*/false)); const auto& finite_storage = internal::checked_cast(*finite->storage_type()); ASSERT_FALSE(finite_storage.field(0)->nullable()); @@ -274,13 +280,15 @@ TEST(RangeType, NonNullableBounds) { // --------------------------------------------------------------------------- // Metadata (Serialize/Deserialize) round-trip -TEST(RangeType, MetadataRoundTrip) { +TEST(FixedClosednessRangeType, MetadataRoundTrip) { using C = extension::RangeClosed; - for (const auto& type : - {extension::range(int32(), C::Left), extension::range(int32(), C::Right), - extension::range(int32(), C::Both), extension::range(int32(), C::Neither), - extension::range(int64(), C::Right), extension::range(date32(), C::Both)}) { - auto rt = checked_pointer_cast(type); + for (const auto& type : {extension::fixed_closedness_range(int32(), C::Left), + extension::fixed_closedness_range(int32(), C::Right), + extension::fixed_closedness_range(int32(), C::Both), + extension::fixed_closedness_range(int32(), C::Neither), + extension::fixed_closedness_range(int64(), C::Right), + extension::fixed_closedness_range(date32(), C::Both)}) { + auto rt = checked_pointer_cast(type); std::string serialized = rt->Serialize(); ASSERT_OK_AND_ASSIGN(auto deserialized, rt->Deserialize(rt->storage_type(), serialized)); @@ -291,7 +299,7 @@ TEST(RangeType, MetadataRoundTrip) { // --------------------------------------------------------------------------- // IPC (BatchRoundTrip) -- registration round-trip -TEST(RangeType, BatchRoundTrip) { +TEST(FixedClosednessRangeType, BatchRoundTrip) { auto type = RangeInt32Right(); auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); @@ -319,13 +327,13 @@ TEST(RangeType, BatchRoundTrip) { } // =========================================================================== -// RangeIncType -- per-value bound inclusivity +// VariableClosednessRangeType -- per-value bound inclusivity // =========================================================================== namespace { -std::shared_ptr IncStorage(const std::shared_ptr& value_type, - bool nullable_bounds = true) { +std::shared_ptr VariableClosednessStorage( + const std::shared_ptr& value_type, bool nullable_bounds = true) { return struct_({field("lower", value_type, nullable_bounds), field("upper", value_type, nullable_bounds), field("lower_inc", boolean(), /*nullable=*/false), @@ -337,16 +345,17 @@ std::shared_ptr IncStorage(const std::shared_ptr& value_type // --------------------------------------------------------------------------- // Basics -TEST(RangeIncType, Basics) { - auto type = - checked_pointer_cast(extension::range_inc(int32())); - ASSERT_EQ("arrow.range_inc", type->extension_name()); +TEST(VariableClosednessRangeType, Basics) { + auto type = checked_pointer_cast( + extension::variable_closedness_range(int32())); + ASSERT_EQ("arrow.variable_closedness_range", type->extension_name()); ASSERT_EQ(*int32(), *type->value_type()); ASSERT_EQ(*type, *type); ASSERT_NE(*arrow::null(), *type); // No type-level parameters: metadata is the empty JSON object. ASSERT_EQ("{}", type->Serialize()); - ASSERT_EQ("extension", type->ToString(false)); + ASSERT_EQ("extension", + type->ToString(false)); // Storage carries the two non-nullable boolean inclusivity fields. const auto& storage = internal::checked_cast(*type->storage_type()); ASSERT_EQ(4, storage.num_fields()); @@ -360,13 +369,15 @@ TEST(RangeIncType, Basics) { // --------------------------------------------------------------------------- // Equals -TEST(RangeIncType, Equals) { - auto i32 = checked_pointer_cast(extension::range_inc(int32())); - auto i32b = - checked_pointer_cast(extension::range_inc(int32())); - auto i64 = checked_pointer_cast(extension::range_inc(int64())); - auto i32_finite = checked_pointer_cast( - extension::range_inc(int32(), /*allow_unbounded=*/false)); +TEST(VariableClosednessRangeType, Equals) { + auto i32 = checked_pointer_cast( + extension::variable_closedness_range(int32())); + auto i32b = checked_pointer_cast( + extension::variable_closedness_range(int32())); + auto i64 = checked_pointer_cast( + extension::variable_closedness_range(int64())); + auto i32_finite = checked_pointer_cast( + extension::variable_closedness_range(int32(), /*allow_unbounded=*/false)); // Same object / same parameters. ASSERT_EQ(*i32, *i32); @@ -378,17 +389,17 @@ TEST(RangeIncType, Equals) { // Different bound nullability is part of storage, hence a different type. ASSERT_NE(*i32, *i32_finite); - // Not equal to non-range types, including a plain arrow.range. + // Not equal to non-range types, including a plain arrow.fixed_closedness_range. ASSERT_NE(*i32, *arrow::int32()); - ASSERT_NE(*i32, *extension::range(int32())); + ASSERT_NE(*i32, *extension::fixed_closedness_range(int32())); } // --------------------------------------------------------------------------- // CreateFromArray -TEST(RangeIncType, CreateFromArray) { - auto type = - checked_pointer_cast(extension::range_inc(int32())); +TEST(VariableClosednessRangeType, CreateFromArray) { + auto type = checked_pointer_cast( + extension::variable_closedness_range(int32())); auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); auto lower_inc = ArrayFromJSON(boolean(), "[true, false, true]"); @@ -404,9 +415,9 @@ TEST(RangeIncType, CreateFromArray) { // --------------------------------------------------------------------------- // Deserialize - valid cases (metadata carries no parameters) -TEST(RangeIncType, DeserializeMetadata) { - auto type = - checked_pointer_cast(extension::range_inc(int32())); +TEST(VariableClosednessRangeType, DeserializeMetadata) { + auto type = checked_pointer_cast( + extension::variable_closedness_range(int32())); // Empty string, empty object, and extra keys are all accepted. for (const auto& serialized : @@ -420,9 +431,9 @@ TEST(RangeIncType, DeserializeMetadata) { // --------------------------------------------------------------------------- // Deserialize - invalid cases -TEST(RangeIncType, DeserializeInvalidMetadata) { - auto type = - checked_pointer_cast(extension::range_inc(int32())); +TEST(VariableClosednessRangeType, DeserializeInvalidMetadata) { + auto type = checked_pointer_cast( + extension::variable_closedness_range(int32())); EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("Invalid serialized JSON data"), @@ -432,9 +443,9 @@ TEST(RangeIncType, DeserializeInvalidMetadata) { type->Deserialize(type->storage_type(), "[]")); } -TEST(RangeIncType, DeserializeInvalidStorage) { - auto type = - checked_pointer_cast(extension::range_inc(int32())); +TEST(VariableClosednessRangeType, DeserializeInvalidStorage) { + auto type = checked_pointer_cast( + extension::variable_closedness_range(int32())); // Not a struct. EXPECT_RAISES_WITH_MESSAGE_THAT(Invalid, testing::HasSubstr("must be a Struct"), @@ -478,21 +489,22 @@ TEST(RangeIncType, DeserializeInvalidStorage) { // --------------------------------------------------------------------------- // Non-nullable bounds -TEST(RangeIncType, NonNullableBounds) { - auto type = - checked_pointer_cast(extension::range_inc(int32())); +TEST(VariableClosednessRangeType, NonNullableBounds) { + auto type = checked_pointer_cast( + extension::variable_closedness_range(int32())); // Both bounds non-nullable: accepted (a finite-only range). ASSERT_OK_AND_ASSIGN( auto from_non_nullable, - type->Deserialize(IncStorage(int32(), /*nullable_bounds=*/false), "{}")); - ASSERT_EQ( - *int32(), - *checked_pointer_cast(from_non_nullable)->value_type()); + type->Deserialize(VariableClosednessStorage(int32(), /*nullable_bounds=*/false), + "{}")); + ASSERT_EQ(*int32(), *checked_pointer_cast( + from_non_nullable) + ->value_type()); // The factory can build non-nullable bounds via allow_unbounded=false. - auto finite = checked_pointer_cast( - extension::range_inc(int32(), /*allow_unbounded=*/false)); + auto finite = checked_pointer_cast( + extension::variable_closedness_range(int32(), /*allow_unbounded=*/false)); const auto& finite_storage = internal::checked_cast(*finite->storage_type()); ASSERT_FALSE(finite_storage.field(0)->nullable()); @@ -505,11 +517,12 @@ TEST(RangeIncType, NonNullableBounds) { // --------------------------------------------------------------------------- // Metadata round-trip -TEST(RangeIncType, MetadataRoundTrip) { - for (const auto& type : - {extension::range_inc(int32()), extension::range_inc(int64()), - extension::range_inc(date32()), extension::range_inc(int32(), false)}) { - auto rt = checked_pointer_cast(type); +TEST(VariableClosednessRangeType, MetadataRoundTrip) { + for (const auto& type : {extension::variable_closedness_range(int32()), + extension::variable_closedness_range(int64()), + extension::variable_closedness_range(date32()), + extension::variable_closedness_range(int32(), false)}) { + auto rt = checked_pointer_cast(type); std::string serialized = rt->Serialize(); ASSERT_OK_AND_ASSIGN(auto deserialized, rt->Deserialize(rt->storage_type(), serialized)); @@ -520,9 +533,9 @@ TEST(RangeIncType, MetadataRoundTrip) { // --------------------------------------------------------------------------- // IPC (BatchRoundTrip) -- registration round-trip -TEST(RangeIncType, BatchRoundTrip) { - auto type = - checked_pointer_cast(extension::range_inc(int32())); +TEST(VariableClosednessRangeType, BatchRoundTrip) { + auto type = checked_pointer_cast( + extension::variable_closedness_range(int32())); auto lower = ArrayFromJSON(int32(), "[1, null, 5]"); auto upper = ArrayFromJSON(int32(), "[10, 20, null]"); auto lower_inc = ArrayFromJSON(boolean(), "[true, false, true]"); diff --git a/cpp/src/arrow/extension_type.cc b/cpp/src/arrow/extension_type.cc index bb673bbc416f..4fa7cae5c767 100644 --- a/cpp/src/arrow/extension_type.cc +++ b/cpp/src/arrow/extension_type.cc @@ -155,10 +155,10 @@ static void CreateGlobalRegistry() { extension::uuid()}; #ifdef ARROW_JSON + ext_types.push_back(extension::fixed_closedness_range(int32())); ext_types.push_back(extension::fixed_shape_tensor(int64(), {})); ext_types.push_back(extension::opaque(null(), "", "")); - ext_types.push_back(extension::range(int32())); - ext_types.push_back(extension::range_inc(int32())); + ext_types.push_back(extension::variable_closedness_range(int32())); ext_types.push_back(extension::variable_shape_tensor(int64(), 0)); #endif diff --git a/docs/source/cpp/api/extension.rst b/docs/source/cpp/api/extension.rst index f2c700e74d78..f23f06978d7d 100644 --- a/docs/source/cpp/api/extension.rst +++ b/docs/source/cpp/api/extension.rst @@ -42,11 +42,11 @@ Extension Type classes :project: arrow_cpp :members: -.. doxygenclass:: arrow::extension::RangeType +.. doxygenclass:: arrow::extension::FixedClosednessRangeType :project: arrow_cpp :members: -.. doxygenclass:: arrow::extension::RangeIncType +.. doxygenclass:: arrow::extension::VariableClosednessRangeType :project: arrow_cpp :members: @@ -69,20 +69,20 @@ Extension Array classes :project: arrow_cpp :members: -.. doxygenclass:: arrow::extension::RangeArray +.. doxygenclass:: arrow::extension::FixedClosednessRangeArray :project: arrow_cpp :members: -.. doxygenclass:: arrow::extension::RangeIncArray +.. doxygenclass:: arrow::extension::VariableClosednessRangeArray :project: arrow_cpp :members: Extension functions =================== -.. doxygenfunction:: arrow::extension::range +.. doxygenfunction:: arrow::extension::fixed_closedness_range :project: arrow_cpp -.. doxygenfunction:: arrow::extension::range_inc +.. doxygenfunction:: arrow::extension::variable_closedness_range :project: arrow_cpp diff --git a/docs/source/format/CanonicalExtensions.rst b/docs/source/format/CanonicalExtensions.rst index cc715b978d6e..6eb785dbdf7e 100644 --- a/docs/source/format/CanonicalExtensions.rst +++ b/docs/source/format/CanonicalExtensions.rst @@ -573,14 +573,15 @@ This extension type is intended to be compatible with ANSI SQL's ``TIMESTAMP WIT It is also *permissible* for the ``offset_minutes`` field to be dictionary-encoded or run-end-encoded. -.. _range_extension: +.. _fixed_closedness_range_extension: -Range -===== +Fixed closedness range +====================== -Range represents a bounded set (mathematical interval) defined by a lower and -an upper bound over an orderable Arrow type T. It is the Arrow equivalent of -PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. +Fixed closedness range represents a bounded set (mathematical interval) +defined by a lower and an upper bound over an orderable Arrow type T. Its +closedness is a type parameter shared by all values. It is the Arrow +equivalent of PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. .. note:: @@ -588,13 +589,14 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. Arrow already has an ``Interval`` type (``INTERVAL_MONTHS``, ``INTERVAL_DAY_TIME``, ``INTERVAL_MONTH_DAY_NANO``) that represents a *duration* -- a signed difference between two points in time. The - ``arrow.range`` extension type is an entirely different concept: it - represents a *bounded set* with explicit lower and upper endpoints, - analogous to a closed or open interval in mathematics. The naming - follows database convention: SQL uses ``INTERVAL`` for durations and - ``RANGE`` (or ``PERIOD``) for bounded sets. + ``arrow.fixed_closedness_range`` and ``arrow.variable_closedness_range`` + extension types are an entirely different concept: they represent a + *bounded set* with explicit lower and upper endpoints, analogous to a + closed or open interval in mathematics. The naming follows database + convention: SQL uses ``INTERVAL`` for durations and ``RANGE`` (or + ``PERIOD``) for bounded sets. -* Extension name: ``arrow.range``. +* Extension name: ``arrow.fixed_closedness_range``. * The storage type of the extension is a ``Struct`` with exactly **two fields, in order**: @@ -661,31 +663,34 @@ PostgreSQL's `range types`_ and SQL:2011 ``PERIOD`` types. .. _range types: https://www.postgresql.org/docs/current/rangetypes.html -.. _range_inc_extension: +.. _variable_closedness_range_extension: -Range Inc -========= +Variable closedness range +========================= -Range Inc represents a bounded set (mathematical interval) over an orderable -Arrow type T whose bound inclusivity is recorded **per value** rather than as a -single type-level parameter. It is the companion of the :ref:`Range -` extension type for ranges that cannot be canonicalized to a -uniform closedness. +Variable closedness range represents a bounded set (mathematical interval) +over an orderable Arrow type T whose bound inclusivity is recorded **per +value** rather than as a single type-level parameter. It is the companion of +the :ref:`fixed closedness range ` extension +type for ranges that cannot be canonicalized to a uniform closedness. .. note:: - **When to use** ``arrow.range`` **vs.** ``arrow.range_inc``. + **When to use** ``arrow.fixed_closedness_range`` **vs.** + ``arrow.variable_closedness_range``. Discrete ranges (e.g. PostgreSQL's ``int4range``, ``int8range``, ``daterange``) canonicalize to a single closedness (left-closed), so they - are best represented by :ref:`arrow.range `, which stores - the closedness once in the type metadata. Continuous ranges (e.g. - PostgreSQL's ``numrange``, ``tsrange``, ``tstzrange``) **cannot** be + are best represented by + :ref:`arrow.fixed_closedness_range `, + which stores the closedness once in the type metadata. Continuous ranges + (e.g. PostgreSQL's ``numrange``, ``tsrange``, ``tstzrange``) **cannot** be canonicalized: two values may share the same endpoints yet differ in - whether those endpoints are included. ``arrow.range_inc`` stores the - inclusivity of each bound alongside the bound itself, mirroring PostgreSQL's - internal range representation, and is the appropriate choice for that case. + whether those endpoints are included. ``arrow.variable_closedness_range`` + stores the inclusivity of each bound alongside the bound itself, mirroring + PostgreSQL's internal range representation, and is the appropriate choice + for that case. -* Extension name: ``arrow.range_inc``. +* Extension name: ``arrow.variable_closedness_range``. * The storage type of the extension is a ``Struct`` with exactly **four fields, in order**: @@ -707,18 +712,21 @@ uniform closedness. storage struct; the subtype is **not** duplicated in the extension metadata. Each of ``lower`` and ``upper`` **may** be nullable, independently of the - other, exactly as in :ref:`arrow.range `: nullability is - only needed to represent an unbounded side. A null bound is **always treated - as exclusive**, regardless of its ``lower_inc`` / ``upper_inc`` flag; positive - and negative infinity can never be included. The ``lower_inc`` and + other, exactly as in + :ref:`arrow.fixed_closedness_range `: + nullability is only needed to represent an unbounded side. A null bound is + **always treated as exclusive**, regardless of its ``lower_inc`` / + ``upper_inc`` flag; positive and negative infinity can never be included. + The ``lower_inc`` and ``upper_inc`` fields are **always non-nullable**. The outer struct's validity bit marks a null/absent range (a missing range, distinct from an empty range). * Extension type parameters: - This type has **no** type-level parameters. Unlike :ref:`arrow.range - `, inclusivity is not fixed by the type; it is carried per - value in the ``lower_inc`` and ``upper_inc`` fields. + This type has **no** type-level parameters. Unlike + :ref:`arrow.fixed_closedness_range `, + inclusivity is not fixed by the type; it is carried per value in the + ``lower_inc`` and ``upper_inc`` fields. For a given value, the range contains every x permitted by its finite bounds and per-value flags: with both flags ``true`` every x such that diff --git a/docs/source/python/api/arrays.rst b/docs/source/python/api/arrays.rst index 5a4dcecdc567..0cb6ebd2a728 100644 --- a/docs/source/python/api/arrays.rst +++ b/docs/source/python/api/arrays.rst @@ -101,7 +101,8 @@ may expose data type-specific methods or properties. JsonArray UuidArray Bool8Array - RangeArray + FixedClosednessRangeArray + VariableClosednessRangeArray .. _api.scalar: @@ -170,4 +171,5 @@ classes may expose data type-specific methods or properties. JsonScalar UuidScalar Bool8Scalar - RangeScalar + FixedClosednessRangeScalar + VariableClosednessRangeScalar diff --git a/docs/source/python/api/datatypes.rst b/docs/source/python/api/datatypes.rst index 2f6125443dd8..88edc83118f8 100644 --- a/docs/source/python/api/datatypes.rst +++ b/docs/source/python/api/datatypes.rst @@ -73,8 +73,8 @@ These should be used to create Arrow data types and schemas. sparse_union opaque bool8 - range_ - range_inc + fixed_closedness_range + variable_closedness_range uuid json_ field @@ -148,8 +148,8 @@ implemented by PyArrow. JsonType UuidType Bool8Type - RangeType - RangeIncType + FixedClosednessRangeType + VariableClosednessRangeType .. _api.types.checking: .. currentmodule:: pyarrow.types diff --git a/docs/source/status.rst b/docs/source/status.rst index a5c229b7b07e..6f4067610715 100644 --- a/docs/source/status.rst +++ b/docs/source/status.rst @@ -133,9 +133,11 @@ Data Types +-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ | Timestamp With Offset | | | ✓ | | | ✓ | | | +-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ -| Range | ✓ | | | | | | | | +| Fixed closedness | ✓ | | | | | | | | +| range | | | | | | | | | +-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ -| Range Inc | ✓ | | | | | | | | +| Variable closedness | ✓ | | | | | | | | +| range | | | | | | | | | +-----------------------+-------+-------+-------+------------+-------+-------+-------+-------+ Notes: diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py index 0b6ad2400e52..f05e5e6ae7dd 100644 --- a/python/pyarrow/__init__.py +++ b/python/pyarrow/__init__.py @@ -165,8 +165,8 @@ def print_entry(label, value): union, sparse_union, dense_union, dictionary, run_end_encoded, - bool8, fixed_shape_tensor, json_, opaque, range_, range_inc, - uuid, + bool8, fixed_closedness_range, fixed_shape_tensor, json_, + opaque, uuid, variable_closedness_range, field, type_for_alias, DataType, DictionaryType, StructType, @@ -177,8 +177,9 @@ def print_entry(label, value): FixedSizeBinaryType, Decimal32Type, Decimal64Type, Decimal128Type, Decimal256Type, BaseExtensionType, ExtensionType, - RunEndEncodedType, Bool8Type, FixedShapeTensorType, - JsonType, OpaqueType, RangeType, RangeIncType, UuidType, + RunEndEncodedType, Bool8Type, FixedClosednessRangeType, + FixedShapeTensorType, JsonType, OpaqueType, UuidType, + VariableClosednessRangeType, UnknownExtensionType, register_extension_type, unregister_extension_type, DictionaryMemo, @@ -214,8 +215,9 @@ def print_entry(label, value): MonthDayNanoIntervalArray, Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array, StructArray, ExtensionArray, - RunEndEncodedArray, Bool8Array, FixedShapeTensorArray, - JsonArray, OpaqueArray, RangeArray, RangeIncArray, UuidArray, + RunEndEncodedArray, Bool8Array, FixedClosednessRangeArray, + FixedShapeTensorArray, JsonArray, OpaqueArray, UuidArray, + VariableClosednessRangeArray, scalar, NA, _NULL as NULL, Scalar, NullScalar, BooleanScalar, Int8Scalar, Int16Scalar, Int32Scalar, Int64Scalar, @@ -233,8 +235,9 @@ def print_entry(label, value): FixedSizeBinaryScalar, DictionaryScalar, MapScalar, StructScalar, UnionScalar, RunEndEncodedScalar, Bool8Scalar, ExtensionScalar, - FixedShapeTensorScalar, JsonScalar, OpaqueScalar, - RangeScalar, RangeIncScalar, UuidScalar) + FixedClosednessRangeScalar, FixedShapeTensorScalar, + JsonScalar, OpaqueScalar, UuidScalar, + VariableClosednessRangeScalar) # Buffers, allocation from pyarrow.lib import (DeviceAllocationType, Device, MemoryManager, diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 09ff72d6cb4f..5d92be3d717f 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -5314,16 +5314,16 @@ cdef class Bool8Array(ExtensionArray): return Bool8Array.from_storage(storage_arr) -cdef class RangeArray(ExtensionArray): +cdef class FixedClosednessRangeArray(ExtensionArray): """ - Concrete class for range extension arrays. + Concrete class for fixed closedness range extension arrays. Examples -------- - Define the extension type for a range array + Define the extension type for a fixed closedness range array >>> import pyarrow as pa - >>> range_type = pa.range_(pa.int32(), "both") + >>> range_type = pa.fixed_closedness_range(pa.int32(), "both") Create an extension array @@ -5332,30 +5332,30 @@ cdef class RangeArray(ExtensionArray): ... range_type.storage_type, ... ) >>> arr = pa.ExtensionArray.from_storage(range_type, storage) - >>> isinstance(arr, pa.RangeArray) + >>> isinstance(arr, pa.FixedClosednessRangeArray) True """ -cdef class RangeIncArray(ExtensionArray): +cdef class VariableClosednessRangeArray(ExtensionArray): """ - Concrete class for range_inc extension arrays. + Concrete class for variable closedness range extension arrays. Examples -------- - Define the extension type for a range_inc array + Define the extension type for a variable closedness range array >>> import pyarrow as pa - >>> range_inc_type = pa.range_inc(pa.float64()) + >>> range_type = pa.variable_closedness_range(pa.float64()) Create an extension array >>> storage = pa.array( ... [{"lower": 1.0, "upper": 5.0, "lower_inc": True, "upper_inc": False}], - ... range_inc_type.storage_type, + ... range_type.storage_type, ... ) - >>> arr = pa.ExtensionArray.from_storage(range_inc_type, storage) - >>> isinstance(arr, pa.RangeIncArray) + >>> arr = pa.ExtensionArray.from_storage(range_type, storage) + >>> isinstance(arr, pa.VariableClosednessRangeArray) True """ diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 9d37703c1d1a..28fb55737ecc 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3142,7 +3142,8 @@ cdef extern from "arrow/extension/range.h" namespace "arrow::extension" nogil: Both Neither - cdef cppclass CRangeType" arrow::extension::RangeType"(CExtensionType): + cdef cppclass CFixedClosednessRangeType \ + " arrow::extension::FixedClosednessRangeType"(CExtensionType): @staticmethod CResult[shared_ptr[CDataType]] Make(shared_ptr[CDataType] value_type, @@ -3152,10 +3153,12 @@ cdef extern from "arrow/extension/range.h" namespace "arrow::extension" nogil: CRangeClosed closed() shared_ptr[CDataType] value_type() - cdef cppclass CRangeArray" arrow::extension::RangeArray"(CExtensionArray): + cdef cppclass CFixedClosednessRangeArray \ + " arrow::extension::FixedClosednessRangeArray"(CExtensionArray): pass - cdef cppclass CRangeIncType" arrow::extension::RangeIncType"(CExtensionType): + cdef cppclass CVariableClosednessRangeType \ + " arrow::extension::VariableClosednessRangeType"(CExtensionType): @staticmethod CResult[shared_ptr[CDataType]] Make(shared_ptr[CDataType] value_type, @@ -3163,7 +3166,8 @@ cdef extern from "arrow/extension/range.h" namespace "arrow::extension" nogil: shared_ptr[CDataType] value_type() - cdef cppclass CRangeIncArray" arrow::extension::RangeIncArray"(CExtensionArray): + cdef cppclass CVariableClosednessRangeArray \ + " arrow::extension::VariableClosednessRangeArray"(CExtensionArray): pass diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd index 3ca1fb836b14..3a17592724ea 100644 --- a/python/pyarrow/lib.pxd +++ b/python/pyarrow/lib.pxd @@ -203,13 +203,13 @@ cdef class Bool8Type(BaseExtensionType): cdef: const CBool8Type* bool8_ext_type -cdef class RangeType(BaseExtensionType): +cdef class FixedClosednessRangeType(BaseExtensionType): cdef: - const CRangeType* range_ext_type + const CFixedClosednessRangeType* range_ext_type -cdef class RangeIncType(BaseExtensionType): +cdef class VariableClosednessRangeType(BaseExtensionType): cdef: - const CRangeIncType* range_inc_ext_type + const CVariableClosednessRangeType* range_ext_type cdef class OpaqueType(BaseExtensionType): cdef: diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi index 6655fe6b279c..f8276806b6c2 100644 --- a/python/pyarrow/public-api.pxi +++ b/python/pyarrow/public-api.pxi @@ -131,10 +131,10 @@ cdef api object pyarrow_wrap_data_type( out = Bool8Type.__new__(Bool8Type) elif extension_name == b"arrow.fixed_shape_tensor": out = FixedShapeTensorType.__new__(FixedShapeTensorType) - elif extension_name == b"arrow.range": - out = RangeType.__new__(RangeType) - elif extension_name == b"arrow.range_inc": - out = RangeIncType.__new__(RangeIncType) + elif extension_name == b"arrow.fixed_closedness_range": + out = FixedClosednessRangeType.__new__(FixedClosednessRangeType) + elif extension_name == b"arrow.variable_closedness_range": + out = VariableClosednessRangeType.__new__(VariableClosednessRangeType) elif extension_name == b"arrow.opaque": out = OpaqueType.__new__(OpaqueType) elif extension_name == b"arrow.uuid": diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi index 5a6c2b509309..bad9cba3e1ec 100644 --- a/python/pyarrow/scalar.pxi +++ b/python/pyarrow/scalar.pxi @@ -1641,14 +1641,14 @@ cdef class Bool8Scalar(ExtensionScalar): return None if py_val is None else py_val != 0 -cdef class RangeScalar(ExtensionScalar): +cdef class FixedClosednessRangeScalar(ExtensionScalar): """ - Concrete class for range extension scalar. + Concrete class for fixed closedness range extension scalar. """ -cdef class RangeIncScalar(ExtensionScalar): +cdef class VariableClosednessRangeScalar(ExtensionScalar): """ - Concrete class for range_inc extension scalar. + Concrete class for variable closedness range extension scalar. """ cdef dict _scalar_classes = { diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py index b8067a9ff34d..4afa93903b05 100644 --- a/python/pyarrow/tests/test_extension_type.py +++ b/python/pyarrow/tests/test_extension_type.py @@ -2086,28 +2086,28 @@ def test_opaque_type(pickle_module, storage_type, storage): (pa.int64(), [{"lower": None, "upper": None}, {"lower": 2, "upper": 8}]), (pa.float64(), [{"lower": 0.0, "upper": 1.5}, None]), ]) -def test_range_type(pickle_module, closed, value_type, bounds): - range_type = pa.range_(value_type, closed) - assert range_type.extension_name == "arrow.range" +def test_fixed_closedness_range_type(pickle_module, closed, value_type, bounds): + range_type = pa.fixed_closedness_range(value_type, closed) + assert range_type.extension_name == "arrow.fixed_closedness_range" assert range_type.value_type == value_type assert range_type.closed == closed assert range_type.storage_type == pa.struct([ pa.field("lower", value_type, nullable=True), pa.field("upper", value_type, nullable=True), ]) - assert "arrow.range" in str(range_type) + assert "arrow.fixed_closedness_range" in str(range_type) # the closed parameter defaults to "left" - assert pa.range_(value_type).closed == "left" + assert pa.fixed_closedness_range(value_type).closed == "left" assert range_type == range_type - assert range_type == pa.range_(value_type, closed) + assert range_type == pa.fixed_closedness_range(value_type, closed) assert range_type != value_type # different closed parameter -> not equal other_closed = "right" if closed != "right" else "left" - assert range_type != pa.range_(value_type, other_closed) + assert range_type != pa.fixed_closedness_range(value_type, other_closed) # different value type -> not equal - assert range_type != pa.range_(pa.decimal128(12, 3), closed) + assert range_type != pa.fixed_closedness_range(pa.decimal128(12, 3), closed) # Pickle roundtrip result = pickle_module.loads(pickle_module.dumps(range_type)) @@ -2125,7 +2125,7 @@ def test_range_type(pickle_module, closed, value_type, bounds): buf = ipc_write_batch(pa.RecordBatch.from_arrays([arr], ["ext"])) batch = ipc_read_batch(buf) - assert batch.column(0).type.extension_name == "arrow.range" + assert batch.column(0).type.extension_name == "arrow.fixed_closedness_range" assert batch.column(0).type.closed == closed assert isinstance(batch.column(0), range_arr_class) assert batch.column(0) == arr @@ -2139,21 +2139,21 @@ def test_range_type(pickle_module, closed, value_type, bounds): assert inner == storage -def test_range_type_invalid_closed(): - with pytest.raises(ValueError, match="Invalid value for range"): - pa.range_(pa.int32(), "invalid") - with pytest.raises(ValueError, match="Invalid value for range"): - pa.range_(pa.int32(), "") +def test_fixed_closedness_range_type_invalid_closed(): + with pytest.raises(ValueError, match="Invalid value for fixed_closedness_range"): + pa.fixed_closedness_range(pa.int32(), "invalid") + with pytest.raises(ValueError, match="Invalid value for fixed_closedness_range"): + pa.fixed_closedness_range(pa.int32(), "") -def test_range_type_allow_unbounded(): +def test_fixed_closedness_range_type_allow_unbounded(): # Default: bounds are nullable (can represent an unbounded / infinite side). - nullable = pa.range_(pa.int32(), "both") + nullable = pa.fixed_closedness_range(pa.int32(), "both") assert nullable.storage_type.field("lower").nullable assert nullable.storage_type.field("upper").nullable # allow_unbounded=False: a finite-only range with non-nullable bounds. - finite = pa.range_(pa.int32(), "both", allow_unbounded=False) + finite = pa.fixed_closedness_range(pa.int32(), "both", allow_unbounded=False) assert not finite.storage_type.field("lower").nullable assert not finite.storage_type.field("upper").nullable assert finite.value_type == pa.int32() @@ -2178,9 +2178,9 @@ def test_range_type_allow_unbounded(): None, ]), ]) -def test_range_inc_type(pickle_module, value_type, rows): - range_type = pa.range_inc(value_type) - assert range_type.extension_name == "arrow.range_inc" +def test_variable_closedness_range_type(pickle_module, value_type, rows): + range_type = pa.variable_closedness_range(value_type) + assert range_type.extension_name == "arrow.variable_closedness_range" assert range_type.value_type == value_type # Storage carries the two bounds plus per-value, non-nullable inclusivity flags. assert range_type.storage_type == pa.struct([ @@ -2189,17 +2189,17 @@ def test_range_inc_type(pickle_module, value_type, rows): pa.field("lower_inc", pa.bool_(), nullable=False), pa.field("upper_inc", pa.bool_(), nullable=False), ]) - assert "arrow.range_inc" in str(range_type) + assert "arrow.variable_closedness_range" in str(range_type) # No type-level closed parameter. assert not hasattr(range_type, "closed") assert range_type == range_type - assert range_type == pa.range_inc(value_type) + assert range_type == pa.variable_closedness_range(value_type) assert range_type != value_type # different value type -> not equal - assert range_type != pa.range_inc(pa.decimal128(12, 3)) - # distinct from a plain arrow.range over the same value type - assert range_type != pa.range_(value_type) + assert range_type != pa.variable_closedness_range(pa.decimal128(12, 3)) + # distinct from arrow.fixed_closedness_range over the same value type + assert range_type != pa.fixed_closedness_range(value_type) # Pickle roundtrip result = pickle_module.loads(pickle_module.dumps(range_type)) @@ -2216,7 +2216,7 @@ def test_range_inc_type(pickle_module, value_type, rows): buf = ipc_write_batch(pa.RecordBatch.from_arrays([arr], ["ext"])) batch = ipc_read_batch(buf) - assert batch.column(0).type.extension_name == "arrow.range_inc" + assert batch.column(0).type.extension_name == "arrow.variable_closedness_range" assert batch.column(0).type.value_type == value_type assert isinstance(batch.column(0), range_arr_class) assert batch.column(0) == arr @@ -2230,9 +2230,9 @@ def test_range_inc_type(pickle_module, value_type, rows): assert inner == storage -def test_range_inc_type_allow_unbounded(): +def test_variable_closedness_range_type_allow_unbounded(): # Default: bounds are nullable (can represent an unbounded / infinite side). - nullable = pa.range_inc(pa.int32()) + nullable = pa.variable_closedness_range(pa.int32()) assert nullable.storage_type.field("lower").nullable assert nullable.storage_type.field("upper").nullable # The inclusivity flags are always non-nullable. @@ -2240,7 +2240,7 @@ def test_range_inc_type_allow_unbounded(): assert not nullable.storage_type.field("upper_inc").nullable # allow_unbounded=False: a finite-only range with non-nullable bounds. - finite = pa.range_inc(pa.int32(), allow_unbounded=False) + finite = pa.variable_closedness_range(pa.int32(), allow_unbounded=False) assert not finite.storage_type.field("lower").nullable assert not finite.storage_type.field("upper").nullable # The flags stay non-nullable regardless of allow_unbounded. @@ -2251,7 +2251,8 @@ def test_range_inc_type_allow_unbounded(): # Distinct types: storage nullability differs. assert finite != nullable - # A non-nullable-bounds range_inc round-trips through its storage. + # A variable closedness range with non-nullable bounds round-trips through + # its storage. storage = pa.array( [{"lower": 1, "upper": 5, "lower_inc": True, "upper_inc": False}], finite.storage_type, diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 322ed0cdb555..d2193b6cb06c 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -2088,28 +2088,28 @@ cdef class Bool8Type(BaseExtensionType): return Bool8Scalar -cdef class RangeType(BaseExtensionType): +cdef class FixedClosednessRangeType(BaseExtensionType): """ - Concrete class for range extension type. + Concrete class for fixed closedness range extension type. - Range represents a bounded set (a mathematical interval) over an orderable - Arrow value type. The underlying storage is a Struct with two fields - "lower" and "upper" of the value type, each optionally nullable; when a - bound field is nullable, a null value denotes an unbounded (infinite) side. - The "closed" parameter controls which finite bounds are inclusive. + A fixed closedness range represents a bounded set (a mathematical interval) + over an orderable Arrow value type. The underlying storage is a Struct with + two fields "lower" and "upper" of the value type, each optionally nullable; + when a bound field is nullable, a null value denotes an unbounded (infinite) + side. The "closed" parameter controls which finite bounds are inclusive. Examples -------- - Create an instance of range extension type: + Create an instance of fixed closedness range extension type: >>> import pyarrow as pa - >>> pa.range_(pa.int32(), "both") - RangeType(extension) + >>> pa.fixed_closedness_range(pa.int32(), "both") + FixedClosednessRangeType(extension) """ cdef void init(self, const shared_ptr[CDataType]& type) except *: BaseExtensionType.init(self, type) - self.range_ext_type = type.get() + self.range_ext_type = type.get() @property def value_type(self): @@ -2135,22 +2135,23 @@ cdef class RangeType(BaseExtensionType): return "neither" def __arrow_ext_class__(self): - return RangeArray + return FixedClosednessRangeArray def __reduce__(self): - return range_, (self.value_type, self.closed) + return fixed_closedness_range, (self.value_type, self.closed) def __arrow_ext_scalar_class__(self): - return RangeScalar + return FixedClosednessRangeScalar -cdef class RangeIncType(BaseExtensionType): +cdef class VariableClosednessRangeType(BaseExtensionType): """ - Concrete class for range_inc extension type. + Concrete class for variable closedness range extension type. - Like :class:`RangeType`, this represents a bounded set (a mathematical - interval) over an orderable Arrow value type, but the inclusivity of each - bound is stored *per value* rather than as a single type-level parameter. + Like :class:`FixedClosednessRangeType`, this represents a bounded set (a + mathematical interval) over an orderable Arrow value type, but the + inclusivity of each bound is stored *per value* rather than as a single + type-level parameter. The underlying storage is a Struct with four fields: "lower" and "upper" (the value type, each optionally nullable to denote an unbounded endpoint) and non-nullable boolean "lower_inc" and "upper_inc" recording whether each @@ -2160,32 +2161,32 @@ cdef class RangeIncType(BaseExtensionType): Examples -------- - Create an instance of range_inc extension type: + Create an instance of variable closedness range extension type: >>> import pyarrow as pa - >>> pa.range_inc(pa.float64()) - RangeIncType(extension) + >>> pa.variable_closedness_range(pa.float64()) + VariableClosednessRangeType(extension) """ cdef void init(self, const shared_ptr[CDataType]& type) except *: BaseExtensionType.init(self, type) - self.range_inc_ext_type = type.get() + self.range_ext_type = type.get() @property def value_type(self): """ The Arrow value type of the "lower" and "upper" bounds. """ - return pyarrow_wrap_data_type(self.range_inc_ext_type.value_type()) + return pyarrow_wrap_data_type(self.range_ext_type.value_type()) def __arrow_ext_class__(self): - return RangeIncArray + return VariableClosednessRangeArray def __reduce__(self): - return range_inc, (self.value_type,) + return variable_closedness_range, (self.value_type,) def __arrow_ext_scalar_class__(self): - return RangeIncScalar + return VariableClosednessRangeScalar cdef class OpaqueType(BaseExtensionType): @@ -5803,9 +5804,10 @@ def bool8(): return out -def range_(DataType value_type not None, str closed="left", allow_unbounded=True): +def fixed_closedness_range(DataType value_type not None, str closed="left", + allow_unbounded=True): """ - Create instance of range extension type. + Create instance of fixed closedness range extension type. Parameters ---------- @@ -5822,12 +5824,12 @@ def range_(DataType value_type not None, str closed="left", allow_unbounded=True Examples -------- - Create an instance of a range extension type: + Create an instance of a fixed closedness range extension type: >>> import pyarrow as pa - >>> type = pa.range_(pa.int32(), "both") + >>> type = pa.fixed_closedness_range(pa.int32(), "both") >>> type - RangeType(extension) + FixedClosednessRangeType(extension) Inspect the data type: @@ -5838,7 +5840,7 @@ def range_(DataType value_type not None, str closed="left", allow_unbounded=True >>> type.storage_type StructType(struct) - Create a range array: + Create a fixed closedness range array: >>> storage = pa.array( ... [{"lower": 1, "upper": 5}, {"lower": None, "upper": 10}], @@ -5846,11 +5848,11 @@ def range_(DataType value_type not None, str closed="left", allow_unbounded=True ... ) >>> arr = pa.ExtensionArray.from_storage(type, storage) >>> arr.type - RangeType(extension) + FixedClosednessRangeType(extension) Returns ------- - type : RangeType + type : FixedClosednessRangeType """ cdef CRangeClosed c_closed @@ -5864,26 +5866,29 @@ def range_(DataType value_type not None, str closed="left", allow_unbounded=True c_closed = CRangeClosed.Neither else: raise ValueError( - f"Invalid value for range \"closed\" parameter: {closed!r}. " - "Expected one of: 'left', 'right', 'both', 'neither'.") + f"Invalid value for fixed_closedness_range \"closed\" parameter: " + f"{closed!r}. Expected one of: 'left', 'right', 'both', 'neither'.") cdef: shared_ptr[CDataType] c_type = GetResultValue( - CRangeType.Make(value_type.sp_type, c_closed, allow_unbounded)) - RangeType out = RangeType.__new__(RangeType) + CFixedClosednessRangeType.Make( + value_type.sp_type, c_closed, allow_unbounded)) + FixedClosednessRangeType out = FixedClosednessRangeType.__new__( + FixedClosednessRangeType) out.init(c_type) return out -def range_inc(DataType value_type not None, allow_unbounded=True): +def variable_closedness_range(DataType value_type not None, allow_unbounded=True): """ - Create instance of range_inc extension type. + Create instance of variable closedness range extension type. - Unlike :func:`range_`, the inclusivity of each bound is stored per value - (in boolean "lower_inc"/"upper_inc" storage fields) rather than as a single - type-level "closed" parameter, so there is no "closed" argument. This is - required for continuous ranges (e.g. PostgreSQL's ``numrange``, ``tsrange``, - ``tstzrange``) that cannot be canonicalized to a uniform closedness. + Unlike :func:`fixed_closedness_range`, the inclusivity of each bound is + stored per value (in boolean "lower_inc"/"upper_inc" storage fields) rather + than as a single type-level "closed" parameter, so there is no "closed" + argument. This is required for continuous ranges (e.g. PostgreSQL's + ``numrange``, ``tsrange``, ``tstzrange``) that cannot be canonicalized to a + uniform closedness. Parameters ---------- @@ -5898,12 +5903,12 @@ def range_inc(DataType value_type not None, allow_unbounded=True): Examples -------- - Create an instance of a range_inc extension type: + Create an instance of a variable closedness range extension type: >>> import pyarrow as pa - >>> type = pa.range_inc(pa.float64()) + >>> type = pa.variable_closedness_range(pa.float64()) >>> type - RangeIncType(extension) + VariableClosednessRangeType(extension) Inspect the data type: @@ -5912,7 +5917,7 @@ def range_inc(DataType value_type not None, allow_unbounded=True): >>> type.storage_type StructType(struct) - Create a range_inc array: + Create a variable closedness range array: >>> storage = pa.array( ... [{"lower": 1.0, "upper": 5.0, "lower_inc": True, "upper_inc": False}], @@ -5920,17 +5925,18 @@ def range_inc(DataType value_type not None, allow_unbounded=True): ... ) >>> arr = pa.ExtensionArray.from_storage(type, storage) >>> arr.type - RangeIncType(extension) + VariableClosednessRangeType(extension) Returns ------- - type : RangeIncType + type : VariableClosednessRangeType """ cdef: shared_ptr[CDataType] c_type = GetResultValue( - CRangeIncType.Make(value_type.sp_type, allow_unbounded)) - RangeIncType out = RangeIncType.__new__(RangeIncType) + CVariableClosednessRangeType.Make(value_type.sp_type, allow_unbounded)) + VariableClosednessRangeType out = VariableClosednessRangeType.__new__( + VariableClosednessRangeType) out.init(c_type) return out