diff --git a/mdio/header_variable_test.cc b/mdio/header_variable_test.cc index 8d1e9ad..2bf1d71 100644 --- a/mdio/header_variable_test.cc +++ b/mdio/header_variable_test.cc @@ -17,7 +17,12 @@ #include #include +#include +#include #include +#include + +#include "mdio/dataset.h" namespace { @@ -135,4 +140,170 @@ TEST(HeaderVariableTest, OperatorPrint) { EXPECT_THAT(ss.str(), ::testing::HasSubstr("segy_file_header")); } +class DatasetMetadataOnlyOpenTest + : public ::testing::TestWithParam { + protected: + static constexpr const char* kHeaderName = "segy_file_header"; + + void SetUp() override { + version_ = GetParam(); + base_path_ = version_ == mdio::zarr::ZarrVersion::kV3 + ? "zarrs/header_open_v3" + : "zarrs/header_open_v2"; + std::filesystem::remove_all(base_path_); + } + + void TearDown() override { std::filesystem::remove_all(base_path_); } + + static nlohmann::json HeaderAttrs() { + return {{"textHeader", "C01 EXAMPLE"}, {"binaryHeader", {{"job_id", 1}}}}; + } + + static void WriteJsonFile(const std::string& path, + const nlohmann::json& json) { + std::filesystem::create_directories( + std::filesystem::path(path).parent_path()); + std::ofstream out(path); + out << json.dump(4); + } + + nlohmann::json SimpleSchema() const { + return nlohmann::json::parse(R"({ + "metadata": { + "name": "header_open_test", + "apiVersion": "1.0.0", + "createdOn": "2023-12-12T15:02:06.413469-06:00" + }, + "variables": [ + { + "name": "data", + "dataType": "float32", + "dimensions": [ + {"name": "x", "size": 8}, + {"name": "y", "size": 8} + ], + "metadata": { + "chunkGrid": { + "name": "regular", + "configuration": { "chunkShape": [8, 8] } + } + } + }, + { + "name": "x", + "dataType": "int32", + "dimensions": [{"name": "x", "size": 8}] + }, + { + "name": "y", + "dataType": "int32", + "dimensions": [{"name": "y", "size": 8}] + } + ] + })"); + } + + void InjectUnsupportedVariable() { + if (version_ == mdio::zarr::ZarrVersion::kV3) { + nlohmann::json zarr_json = { + {"zarr_format", 3}, + {"node_type", "array"}, + {"shape", nlohmann::json::array()}, + {"data_type", + {{"name", "fixed_length_utf32"}, + {"configuration", {{"length_bytes", 4}}}}}, + {"chunk_grid", + {{"name", "regular"}, + {"configuration", {{"chunk_shape", nlohmann::json::array()}}}}}, + {"chunk_key_encoding", + {{"name", "default"}, {"configuration", {{"separator", "/"}}}}}, + {"fill_value", nullptr}, + {"codecs", nlohmann::json::array({{{"name", "bytes"}}})}, + {"attributes", HeaderAttrs()}}; + WriteJsonFile(base_path_ + "/" + kHeaderName + "/zarr.json", zarr_json); + return; + } + + nlohmann::json zarray = {{"zarr_format", 2}, + {"shape", nlohmann::json::array()}, + {"chunks", nlohmann::json::array()}, + {"dtype", "> zmetadata; + } + zmetadata.at("metadata")[std::string(kHeaderName) + "/.zarray"] = zarray; + zmetadata.at("metadata")[std::string(kHeaderName) + "/.zattrs"] = + HeaderAttrs(); + WriteJsonFile(zmetadata_path, zmetadata); + } + + mdio::zarr::ZarrVersion version_; + std::string base_path_; +}; + +TEST_P(DatasetMetadataOnlyOpenTest, OpenKeepsUnsupportedVariableAsMetadata) { + auto schema = SimpleSchema(); + auto created = mdio::Dataset::from_json(schema, base_path_, version_, + mdio::constants::kCreateClean); + ASSERT_TRUE(created.status().ok()) << created.status(); + + InjectUnsupportedVariable(); + + auto opened = mdio::Dataset::Open(base_path_, mdio::constants::kOpen); + ASSERT_TRUE(opened.status().ok()) << opened.status(); + auto dataset = opened.value(); + + EXPECT_TRUE(dataset.variables.contains_key("data")); + EXPECT_FALSE(dataset.variables.contains_key(kHeaderName)); + ASSERT_TRUE(dataset.header_variables.contains_key(kHeaderName)); + + auto data = dataset.variables.get("data"); + ASSERT_TRUE(data.status().ok()) << data.status(); + auto data_read = data.value().Read(); + ASSERT_TRUE(data_read.status().ok()) << data_read.status(); + + auto header = dataset.get_header_variable(kHeaderName); + ASSERT_TRUE(header.status().ok()) << header.status(); + EXPECT_EQ(header.value().get_variable_name(), kHeaderName); + EXPECT_EQ(header.value().rank(), 0); + EXPECT_FALSE(header.value().Read().status().ok()); + EXPECT_EQ(header.value().GetAttributes()["attributes"]["textHeader"], + "C01 EXAMPLE"); + + std::stringstream printed; + printed << dataset; + EXPECT_THAT(printed.str(), + ::testing::HasSubstr("Header Variable: segy_file_header")); + + nlohmann::json updated = header.value().GetAttributes(); + updated["attributes"]["testMarker"] = "cpp-mdio"; + ASSERT_TRUE(header.value().UpdateAttributes(updated).ok()); + auto commit = dataset.CommitMetadata(); + ASSERT_TRUE(commit.status().ok()) << commit.status(); + + auto reopened = mdio::Dataset::Open(base_path_, mdio::constants::kOpen); + ASSERT_TRUE(reopened.status().ok()) << reopened.status(); + auto reopened_header = reopened.value().get_header_variable(kHeaderName); + ASSERT_TRUE(reopened_header.status().ok()) << reopened_header.status(); + EXPECT_EQ(reopened_header.value().GetAttributes()["attributes"]["testMarker"], + "cpp-mdio"); +} + +INSTANTIATE_TEST_SUITE_P( + ZarrVersions, DatasetMetadataOnlyOpenTest, + ::testing::Values(mdio::zarr::ZarrVersion::kV2, + mdio::zarr::ZarrVersion::kV3), + [](const ::testing::TestParamInfo& info) { + return info.param == mdio::zarr::ZarrVersion::kV3 ? "V3" : "V2"; + }); + } // namespace diff --git a/mdio/zarr/zarr_v2.h b/mdio/zarr/zarr_v2.h index 194c616..700744d 100644 --- a/mdio/zarr/zarr_v2.h +++ b/mdio/zarr/zarr_v2.h @@ -124,8 +124,8 @@ inline std::vector GetStructFieldNames( * Mirrors the Zarr V3 behavior: Python `mdio` stores some variables (e.g. the * SEG-Y file header) using string/bytes/datetime dtypes whose content lives in * the variable attributes rather than in chunk data. TensorStore's zarr driver - * cannot open these, so they are treated as metadata-only and skipped during - * discovery instead of failing the whole dataset open. + * cannot open these, so they are treated as metadata-only header variables + * during discovery instead of failing the whole dataset open. * * Zarr V2 dtypes are numpy typestrings such as "