diff --git a/CHANGELOG.md b/CHANGELOG.md index 098f1a172..46fdbb1ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,14 @@ ### Bug Fixes +- **[client-v2]** Fixed reading a `Variant`, `Nested`, `Decimal` or `Enum` value held in a `Dynamic` column. The + concrete type rebuilt from the binary type encoding did not match what the server encoded: `Variant` was wrapped + twice (so the discriminator selected the wrong element), `Nested` read only the element names and left the element + type encodings in the stream, and `Decimal`/`Enum` lost their precision and scale / their constants whenever the + value sat inside another type, so a decimal read back unscaled (`1.2500` as `12500`) and every enum value read back + as ``. The constant width of an enum is now taken from the type tag rather than from the number of + constants, which also fixes reading an `Enum16` with fewer than 128 constants and negative `Enum8` constants. + (https://github.com/ClickHouse/clickhouse-java/issues/3003) - **[client-v2]** Fixed truncated LZ4 stream errors reporting literal `{0}` and `{1}` placeholders instead of the number of bytes read and expected. (https://github.com/ClickHouse/clickhouse-java/issues/3108) - **[jdbc-v2]** Fixed `DatabaseMetaData#getTables` reporting `TABLE_TYPE = TABLE` for a table with the `BigQuery` diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 4276ef941..fce84b9a3 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -5,6 +5,7 @@ import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseEnum; +import com.clickhouse.data.ClickHouseUtils; import com.clickhouse.data.value.ClickHouseBitmap; import org.slf4j.Logger; import org.slf4j.helpers.NOPLogger; @@ -1540,7 +1541,9 @@ private ClickHouseColumn readDynamicData() throws IOException { case Decimal256: { int precision = readByte(); int scale = readByte(); - return ClickHouseColumn.of("v", ClickHouseDataType.binTag2Type.get(tag), false, precision, scale); + // Rendered the way the server renders a decimal type, so that a parent type encoding + // (array, tuple, map, ...) can append it and keep the precision and the scale. + return ClickHouseColumn.of("v", "Decimal(" + precision + ", " + scale + ")"); } case Dynamic: { int maxTypes = readVarInt(input); @@ -1552,16 +1555,24 @@ private ClickHouseColumn readDynamicData() throws IOException { int constants = readVarInt(input); int[] values = new int[constants]; String[] names = new String[constants]; - ClickHouseDataType enumType = constants > 127 ? ClickHouseDataType.Enum16 : ClickHouseDataType.Enum8; + // The width of the constant values is defined by the type tag, not by their number: + // Enum8 encodes them as Int8 and Enum16 as Int16. + ClickHouseDataType enumType = type == ClickHouseDataType.Enum16 ? + ClickHouseDataType.Enum16 : ClickHouseDataType.Enum8; + StringBuilder enumDef = new StringBuilder(SB_INIT_SIZE); + enumDef.append(enumType.name()).append('('); for (int i = 0; i < constants; i++) { names[i] = readString(input); - if (enumType == ClickHouseDataType.Enum8) { - values[i] = readUnsignedByte(); - } else { - values[i] = readUnsignedShortLE(); + values[i] = enumType == ClickHouseDataType.Enum8 ? readByte() : readShortLE(); + if (i > 0) { + enumDef.append(", "); } + enumDef.append('\'').append(ClickHouseUtils.escape(names[i], '\'')).append("' = ").append(values[i]); } - return new ClickHouseColumn(enumType, "v", enumType.name(), false, false, Collections.emptyList(), Collections.emptyList(), + enumDef.append(')'); + // The constants are a part of the type: a parent type encoding appends this type name, + // so it has to carry them to keep the names of the values readable. + return new ClickHouseColumn(enumType, "v", enumDef.toString(), false, false, Collections.emptyList(), Collections.emptyList(), new ClickHouseEnum(names, values)); } case FixedString: { @@ -1624,8 +1635,10 @@ private ClickHouseColumn readDynamicData() throws IOException { StringBuilder nested = new StringBuilder(SB_INIT_SIZE); nested.append("Nested("); for (int i = 0; i < size; i++) { - String name = readString(input); - nested.append(name).append(','); + // Nested is encoded as a named tuple: every element name is followed by the + // type encoding of that element, which has to be consumed here as well. + nested.append(readString(input)).append(' '); + nested.append(readDynamicData().getOriginalTypeName()).append(','); } nested.setLength(nested.length() - 1); nested.append(')'); @@ -1680,7 +1693,7 @@ private ClickHouseColumn readDynamicData() throws IOException { } variant.setLength(variant.length() - 1); variant.append(")"); - return ClickHouseColumn.of("v", "Variant(" + variant + ")"); + return ClickHouseColumn.of("v", variant.toString()); } case AggregateFunction: throw new ClientException("Aggregate functions are not supported yet"); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java index 521bf4ed7..0f46013b7 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java @@ -1,460 +1,587 @@ -package com.clickhouse.client.api.data_formats.internal; - -import com.clickhouse.client.api.ClientException; -import com.clickhouse.data.ClickHouseColumn; -import com.clickhouse.data.ClickHouseDataType; -import com.clickhouse.data.format.BinaryStreamUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.math.BigInteger; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.TimeZone; - -import org.testng.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -public class BinaryStreamReaderTests { - - private ZoneId tzLAX; - private ZoneId tzBER; - - @BeforeClass - void beforeClass() { - tzLAX = ZoneId.of("America/Los_Angeles"); - tzBER = ZoneId.of("Europe/Berlin"); - } - - @Test - public void testCachedByteAllocator() { - BinaryStreamReader.CachingByteBufferAllocator allocator = new BinaryStreamReader.CachingByteBufferAllocator(); - - for (int i = 0; i < 6; i++) { - int size = (int) Math.pow(2, i); - byte[] firstAllocation = allocator.allocate(size); - byte[] nextAllocation = allocator.allocate(size); - Assert.assertTrue(firstAllocation == nextAllocation, "Should be the same buffer for size " + size); - } - - for (int i = 6; i < 16; i++) { - int size = (int) Math.pow(2, i); - byte[] firstAllocation = allocator.allocate(size); - byte[] nextAllocation = allocator.allocate(size); - Assert.assertNotSame(firstAllocation, nextAllocation); - } - } - - @Test(dataProvider = "dateTestData") - void readDateZonedDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - /* - * Date is number of days since 1970-01-01 (unsigned) - * ... The date value is stored without the time zone. - */ - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDate(baos, zdt, writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDate( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)), - expectedZDT); - } - - @Test(dataProvider = "dateTestData") - void readDateOffsetDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDate(baos, zdt.toOffsetDateTime(), writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDate( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), - expectedZDT.toOffsetDateTime()); - } - - @DataProvider(name = "dateTestData") - private Object[][] provideDateTestData() { - ZonedDateTime zdtLAX = ZonedDateTime.of( - 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); - ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); - return new Object[][] { - // no conversion at all - { zdtLAX, tzLAX, tzLAX, zdtLAX.truncatedTo(ChronoUnit.DAYS) }, - - // write using Berlin local date -> next day - { zdtLAX, tzBER, tzBER, zdtLAX.plusDays(1L).withZoneSameLocal(tzBER) - .truncatedTo(ChronoUnit.DAYS) }, - - // read using different time zone: local date same as original - { zdtLAX, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) - .truncatedTo(ChronoUnit.DAYS) }, - - // write using different time zone: local date same as original - { zdtBER, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) - .truncatedTo(ChronoUnit.DAYS) } - }; - - } - - @Test(dataProvider = "dateTimeTestData") - void readDateTime32ZonedDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDateTime32(baos, zdt, writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDateTime32( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)), - expectedZDT.truncatedTo(ChronoUnit.SECONDS)); - } - - @Test(dataProvider = "dateTimeTestData") - void readDateTime32OffsetDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDateTime32(baos, zdt.toOffsetDateTime(), writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDateTime32( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), - expectedZDT.toOffsetDateTime().truncatedTo(ChronoUnit.SECONDS)); - } - - @Test(dataProvider = "dateTimeTestData") - void readDateTime32Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDateTime32(baos, zdt.toInstant(), writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDateTime32( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)), - expectedZDT.truncatedTo(ChronoUnit.SECONDS)); - } - - @Test(dataProvider = "dateTimeTestData") - void readDateTime64Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDateTime64(baos, zdt.toInstant(), 9, writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDateTime64( - new ByteArrayInputStream(bytes), - bytes, - 9, - TimeZone.getTimeZone(readTZ)), - expectedZDT); - } - - @DataProvider(name = "dateTimeTestData") - private Object[][] provideDateTimeTestData() { - ZonedDateTime zdtLAX = ZonedDateTime.of( - 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); - ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); - return new Object[][] { - { zdtLAX, tzLAX, tzLAX, zdtLAX }, - { zdtLAX, tzBER, tzLAX, zdtLAX }, - { zdtLAX, tzLAX, tzBER, zdtBER }, - { zdtBER, tzLAX, tzBER, zdtBER } - }; - } - - @Test - public void testArrayValue() throws Exception { - BinaryStreamReader.ArrayValue array = new BinaryStreamReader.ArrayValue(int.class, 10); - - for (int i = 0; i < array.length(); i++) { - array.set(i, i); - } - - int[] array1 = (int[]) array.getArray(); - Object[] array2 = array.getArrayOfObjects(); - Assert.assertEquals(array1.length, array2.length); - } - - @Test - public void testDynamicSimpleAggregateFunctionConsumesWholeTypeEncoding() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - baos.write(ClickHouseDataType.SimpleAggregateFunction.getBinTag()); - BinaryStreamUtils.writeString(baos, "sum"); - BinaryStreamUtils.writeVarInt(baos, 0); - BinaryStreamUtils.writeVarInt(baos, 1); - baos.write(ClickHouseDataType.UInt64.getBinTag()); - BinaryStreamUtils.writeUnsignedInt64(baos, 42); - BinaryStreamUtils.writeInt32(baos, 4242); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - Assert.assertEquals(reader.readValue(ClickHouseColumn.of("v", "Dynamic")), BigInteger.valueOf(42)); - Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); - } - - @Test(expectedExceptions = ClientException.class) - public void testDynamicParameterizedSimpleAggregateFunctionRejected() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - baos.write(ClickHouseDataType.SimpleAggregateFunction.getBinTag()); - BinaryStreamUtils.writeString(baos, "sum"); - BinaryStreamUtils.writeVarInt(baos, 1); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - reader.readValue(ClickHouseColumn.of("v", "Dynamic")); - } - - @Test - public void testReadNullVariantReturnsNull() throws Exception { - ClickHouseColumn column = ClickHouseColumn.of("v", "Variant(Int32, String)"); - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(new byte[]{(byte) 0xFF}), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - Assert.assertNull(reader.readValue(column)); - } - - @Test - public void testNullableArrayValueUsesBoxedComponentType() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryStreamUtils.writeVarInt(baos, 2); - BinaryStreamUtils.writeNonNull(baos); - BinaryStreamUtils.writeFloat64(baos, 1.0); - BinaryStreamUtils.writeNonNull(baos); - BinaryStreamUtils.writeFloat64(baos, 2.0); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( - ClickHouseColumn.of("v", "Array(Nullable(Float64))")); - - Assert.assertEquals(array.getArray().getClass().getComponentType(), Double.class); - } - - @Test - public void testNullableUnsignedArrayUsesWidenedType() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryStreamUtils.writeVarInt(baos, 2); - BinaryStreamUtils.writeNonNull(baos); - BinaryStreamUtils.writeUnsignedInt8(baos, 10); - BinaryStreamUtils.writeNonNull(baos); - BinaryStreamUtils.writeUnsignedInt8(baos, 20); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( - ClickHouseColumn.of("v", "Array(Nullable(UInt8))")); - - Assert.assertEquals(array.getArray().getClass().getComponentType(), Short.class); - } - - @Test - public void testNullableEnumArrayUsesEnumValueType() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryStreamUtils.writeVarInt(baos, 2); - BinaryStreamUtils.writeNonNull(baos); - baos.write(1); // enum ordinal for 'a' - BinaryStreamUtils.writeNonNull(baos); - baos.write(2); // enum ordinal for 'b' - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( - ClickHouseColumn.of("v", "Array(Nullable(Enum8('a'=1,'b'=2)))")); - - Assert.assertEquals(array.getArray().getClass().getComponentType(), - BinaryStreamReader.EnumValue.class); - } - - @Test - public void testEmptyArrayTypes() throws Exception { - assertEmptyArrayComponentType("Array(UInt8)", short.class); - assertEmptyArrayComponentType("Array(Nullable(UInt8))", Short.class); - assertEmptyArrayComponentType("Array(String)", String.class); - assertEmptyArrayComponentType("Array(Nullable(String))", String.class); - assertEmptyArrayComponentType("Array(Enum8('a'=1))", BinaryStreamReader.EnumValue.class); - assertEmptyArrayComponentType("Array(Nullable(Enum8('a'=1)))", BinaryStreamReader.EnumValue.class); - assertEmptyArrayComponentType("Array(Variant(Int32, String))", Object.class); - assertEmptyArrayComponentType("Array(Array(String))", BinaryStreamReader.ArrayValue.class); - } - - private void assertEmptyArrayComponentType(String columnType, Class expectedComponentType) throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryStreamUtils.writeVarInt(baos, 0); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( - ClickHouseColumn.of("v", columnType)); - - Assert.assertEquals(array.getArray().getClass().getComponentType(), expectedComponentType, "Failed for " + columnType); - } - - // Native-format QBit column payloads captured from ClickHouse 26.5.1 (the block header stripped, - // leaving the Tuple(FixedString) bit-plane bytes readQBitNative consumes) paired with the vector the - // server encoded. Pins the decode against the server's real layout as a unit test, because the QBit - // integration tests skip on the coverage build's older server. - @DataProvider(name = "qbitNativeGoldenBytes") - public static Object[][] qbitNativeGoldenBytes() { - return new Object[][] { - {"Float32 dim 3", "QBit(Float32, 3)", 1, - "0206010101010101010404000000000000000000000000000000000000000000", - false, new Object[] {new float[] {1f, -2f, 3.5f}}}, - {"Float32 dim 8 (full single-byte plane)", "QBit(Float32, 8)", 1, - "02fe010101010181796454000000000000000000000000000000000000000000", - false, new Object[] {new float[] {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}}}, - {"Float32 dim 10 (two-byte plane, partial last byte)", "QBit(Float32, 10)", 1, - "02aa03fe000100010001000100010381007900640254010000000000000000000000000000000000000000000000000000000000000000000000000000000000", - false, new Object[] {new float[] {1f, -2f, 3.5f, -4f, 5f, -6f, 7f, -8f, 9f, -10f}}}, - {"Float32 dim 1", "QBit(Float32, 1)", 1, - "0001000000000100000001000100010000000000000000000000000000000000", - false, new Object[] {new float[] {42.5f}}}, - {"Float64 dim 3 (64 bit planes)", "QBit(Float64, 3)", 1, - "02060101010101010101010104040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - true, new Object[] {new double[] {1d, -2d, 3.5d}}}, - {"BFloat16 dim 8 (16 bit planes)", "QBit(BFloat16, 8)", 1, - "00fe01010101e1995500000000000000", - false, new Object[] {new float[] {1f, 2f, 4f, 8f, 16f, 32f, 64f, 128f}}}, - {"Float32 dim 3, three rows (column-major slicing)", "QBit(Float32, 3)", 3, - "000000060707010000010000010000010000010000010006010701040401000201000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - false, new Object[] { - new float[] {1f, 2f, 3f}, new float[] {4f, 5f, 6f}, new float[] {7f, 8f, 9f}}}, - }; - } - - @Test(dataProvider = "qbitNativeGoldenBytes") - public void testReadQBitNativeDecodesGoldenBytes(String label, String columnType, int nRows, - String columnDataHex, boolean isDouble, Object[] expectedRows) throws Exception { - List rows = qbitReader(fromHex(columnDataHex)) - .readQBitNative(ClickHouseColumn.of("vec", columnType), nRows); - - Assert.assertEquals(rows.size(), nRows, label); - for (int r = 0; r < nRows; r++) { - Object vector = ((BinaryStreamReader.ArrayValue) rows.get(r)).getArray(); - if (isDouble) { - Assert.assertEquals((double[]) vector, (double[]) expectedRows[r], label + " row " + r); - } else { - Assert.assertEquals((float[]) vector, (float[]) expectedRows[r], label + " row " + r); - } - } - } - - @Test - public void testReadQBitNativeDecodesSpecialFloatValues() throws Exception { - // NaN, +Inf, -Inf and -0.0 must survive the bit-plane transpose bit-for-bit. - float[] vector = (float[]) ((BinaryStreamReader.ArrayValue) qbitReader(fromHex( - "0c07070707070707070100000000000000000000000000000000000000000000")) - .readQBitNative(ClickHouseColumn.of("vec", "QBit(Float32, 4)"), 1).get(0)).getArray(); - - float[] expected = {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, -0.0f}; - Assert.assertEquals(vector.length, expected.length); - for (int j = 0; j < expected.length; j++) { - Assert.assertEquals(Float.floatToRawIntBits(vector[j]), Float.floatToRawIntBits(expected[j]), - "element " + j); - } - } - - @Test - public void testReadQBitNativeRejectsUnsupportedElementType() { - // readQBitNative reconstructs only float element types; a non-float element is rejected by its own - // guard (the Native reader also filters these earlier), which fires before any read. - ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Int8, 3)"); - ClientException ex = Assert.expectThrows(ClientException.class, - () -> qbitReader(new byte[0]).readQBitNative(column, 1)); - Assert.assertTrue(ex.getMessage().contains("Float32"), - "Expected an unsupported-element-type message, got: " + ex.getMessage()); - } - - @Test - public void testReadQBitNativeRejectsIntOverflowPlaneSize() { - // nRows * ceil(dimension/8) per bit plane must not overflow a 32-bit int; the reader rejects it - // before allocating or reading, so an empty stream is sufficient (200000 -> 25000 bytes/plane, - // 90000 rows -> 2_250_000_000 bytes > Integer.MAX_VALUE). - ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Float32, 200000)"); - ClientException ex = Assert.expectThrows(ClientException.class, - () -> qbitReader(new byte[0]).readQBitNative(column, 90000)); - Assert.assertTrue(ex.getMessage().contains("too large"), - "Expected an overflow rejection message, got: " + ex.getMessage()); - } - - private static BinaryStreamReader qbitReader(byte[] columnData) { - return new BinaryStreamReader(new ByteArrayInputStream(columnData), TimeZone.getTimeZone("UTC"), - null, new BinaryStreamReader.CachingByteBufferAllocator(), false, null, false); - } - - private static byte[] fromHex(String hex) { - byte[] out = new byte[hex.length() / 2]; - for (int i = 0; i < out.length; i++) { - out[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); - } - return out; - } -} +package com.clickhouse.client.api.data_formats.internal; + +import com.clickhouse.client.api.ClientException; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.format.BinaryStreamUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.TimeZone; + +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class BinaryStreamReaderTests { + + private ZoneId tzLAX; + private ZoneId tzBER; + + @BeforeClass + void beforeClass() { + tzLAX = ZoneId.of("America/Los_Angeles"); + tzBER = ZoneId.of("Europe/Berlin"); + } + + @Test + public void testCachedByteAllocator() { + BinaryStreamReader.CachingByteBufferAllocator allocator = new BinaryStreamReader.CachingByteBufferAllocator(); + + for (int i = 0; i < 6; i++) { + int size = (int) Math.pow(2, i); + byte[] firstAllocation = allocator.allocate(size); + byte[] nextAllocation = allocator.allocate(size); + Assert.assertTrue(firstAllocation == nextAllocation, "Should be the same buffer for size " + size); + } + + for (int i = 6; i < 16; i++) { + int size = (int) Math.pow(2, i); + byte[] firstAllocation = allocator.allocate(size); + byte[] nextAllocation = allocator.allocate(size); + Assert.assertNotSame(firstAllocation, nextAllocation); + } + } + + @Test(dataProvider = "dateTestData") + void readDateZonedDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + /* + * Date is number of days since 1970-01-01 (unsigned) + * ... The date value is stored without the time zone. + */ + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDate(baos, zdt, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDate( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT); + } + + @Test(dataProvider = "dateTestData") + void readDateOffsetDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDate(baos, zdt.toOffsetDateTime(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDate( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), + expectedZDT.toOffsetDateTime()); + } + + @DataProvider(name = "dateTestData") + private Object[][] provideDateTestData() { + ZonedDateTime zdtLAX = ZonedDateTime.of( + 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); + ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); + return new Object[][] { + // no conversion at all + { zdtLAX, tzLAX, tzLAX, zdtLAX.truncatedTo(ChronoUnit.DAYS) }, + + // write using Berlin local date -> next day + { zdtLAX, tzBER, tzBER, zdtLAX.plusDays(1L).withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) }, + + // read using different time zone: local date same as original + { zdtLAX, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) }, + + // write using different time zone: local date same as original + { zdtBER, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) } + }; + + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32ZonedDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32OffsetDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt.toOffsetDateTime(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), + expectedZDT.toOffsetDateTime().truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt.toInstant(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime64Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime64(baos, zdt.toInstant(), 9, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime64( + new ByteArrayInputStream(bytes), + bytes, + 9, + TimeZone.getTimeZone(readTZ)), + expectedZDT); + } + + @DataProvider(name = "dateTimeTestData") + private Object[][] provideDateTimeTestData() { + ZonedDateTime zdtLAX = ZonedDateTime.of( + 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); + ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); + return new Object[][] { + { zdtLAX, tzLAX, tzLAX, zdtLAX }, + { zdtLAX, tzBER, tzLAX, zdtLAX }, + { zdtLAX, tzLAX, tzBER, zdtBER }, + { zdtBER, tzLAX, tzBER, zdtBER } + }; + } + + @Test + public void testArrayValue() throws Exception { + BinaryStreamReader.ArrayValue array = new BinaryStreamReader.ArrayValue(int.class, 10); + + for (int i = 0; i < array.length(); i++) { + array.set(i, i); + } + + int[] array1 = (int[]) array.getArray(); + Object[] array2 = array.getArrayOfObjects(); + Assert.assertEquals(array1.length, array2.length); + } + + @Test + public void testDynamicSimpleAggregateFunctionConsumesWholeTypeEncoding() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.SimpleAggregateFunction.getBinTag()); + BinaryStreamUtils.writeString(baos, "sum"); + BinaryStreamUtils.writeVarInt(baos, 0); + BinaryStreamUtils.writeVarInt(baos, 1); + baos.write(ClickHouseDataType.UInt64.getBinTag()); + BinaryStreamUtils.writeUnsignedInt64(baos, 42); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("v", "Dynamic")), BigInteger.valueOf(42)); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test(expectedExceptions = ClientException.class) + public void testDynamicParameterizedSimpleAggregateFunctionRejected() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.SimpleAggregateFunction.getBinTag()); + BinaryStreamUtils.writeString(baos, "sum"); + BinaryStreamUtils.writeVarInt(baos, 1); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + reader.readValue(ClickHouseColumn.of("v", "Dynamic")); + } + + @Test + public void testReadNullVariantReturnsNull() throws Exception { + ClickHouseColumn column = ClickHouseColumn.of("v", "Variant(Int32, String)"); + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(new byte[]{(byte) 0xFF}), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + Assert.assertNull(reader.readValue(column)); + } + + @Test + public void testNullableArrayValueUsesBoxedComponentType() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeFloat64(baos, 1.0); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeFloat64(baos, 2.0); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( + ClickHouseColumn.of("v", "Array(Nullable(Float64))")); + + Assert.assertEquals(array.getArray().getClass().getComponentType(), Double.class); + } + + @Test + public void testNullableUnsignedArrayUsesWidenedType() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeUnsignedInt8(baos, 10); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeUnsignedInt8(baos, 20); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( + ClickHouseColumn.of("v", "Array(Nullable(UInt8))")); + + Assert.assertEquals(array.getArray().getClass().getComponentType(), Short.class); + } + + @Test + public void testNullableEnumArrayUsesEnumValueType() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeNonNull(baos); + baos.write(1); // enum ordinal for 'a' + BinaryStreamUtils.writeNonNull(baos); + baos.write(2); // enum ordinal for 'b' + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( + ClickHouseColumn.of("v", "Array(Nullable(Enum8('a'=1,'b'=2)))")); + + Assert.assertEquals(array.getArray().getClass().getComponentType(), + BinaryStreamReader.EnumValue.class); + } + + @Test + public void testEmptyArrayTypes() throws Exception { + assertEmptyArrayComponentType("Array(UInt8)", short.class); + assertEmptyArrayComponentType("Array(Nullable(UInt8))", Short.class); + assertEmptyArrayComponentType("Array(String)", String.class); + assertEmptyArrayComponentType("Array(Nullable(String))", String.class); + assertEmptyArrayComponentType("Array(Enum8('a'=1))", BinaryStreamReader.EnumValue.class); + assertEmptyArrayComponentType("Array(Nullable(Enum8('a'=1)))", BinaryStreamReader.EnumValue.class); + assertEmptyArrayComponentType("Array(Variant(Int32, String))", Object.class); + assertEmptyArrayComponentType("Array(Array(String))", BinaryStreamReader.ArrayValue.class); + } + + private void assertEmptyArrayComponentType(String columnType, Class expectedComponentType) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryStreamUtils.writeVarInt(baos, 0); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( + ClickHouseColumn.of("v", columnType)); + + Assert.assertEquals(array.getArray().getClass().getComponentType(), expectedComponentType, "Failed for " + columnType); + } + + // Native-format QBit column payloads captured from ClickHouse 26.5.1 (the block header stripped, + // leaving the Tuple(FixedString) bit-plane bytes readQBitNative consumes) paired with the vector the + // server encoded. Pins the decode against the server's real layout as a unit test, because the QBit + // integration tests skip on the coverage build's older server. + @DataProvider(name = "qbitNativeGoldenBytes") + public static Object[][] qbitNativeGoldenBytes() { + return new Object[][] { + {"Float32 dim 3", "QBit(Float32, 3)", 1, + "0206010101010101010404000000000000000000000000000000000000000000", + false, new Object[] {new float[] {1f, -2f, 3.5f}}}, + {"Float32 dim 8 (full single-byte plane)", "QBit(Float32, 8)", 1, + "02fe010101010181796454000000000000000000000000000000000000000000", + false, new Object[] {new float[] {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}}}, + {"Float32 dim 10 (two-byte plane, partial last byte)", "QBit(Float32, 10)", 1, + "02aa03fe000100010001000100010381007900640254010000000000000000000000000000000000000000000000000000000000000000000000000000000000", + false, new Object[] {new float[] {1f, -2f, 3.5f, -4f, 5f, -6f, 7f, -8f, 9f, -10f}}}, + {"Float32 dim 1", "QBit(Float32, 1)", 1, + "0001000000000100000001000100010000000000000000000000000000000000", + false, new Object[] {new float[] {42.5f}}}, + {"Float64 dim 3 (64 bit planes)", "QBit(Float64, 3)", 1, + "02060101010101010101010104040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + true, new Object[] {new double[] {1d, -2d, 3.5d}}}, + {"BFloat16 dim 8 (16 bit planes)", "QBit(BFloat16, 8)", 1, + "00fe01010101e1995500000000000000", + false, new Object[] {new float[] {1f, 2f, 4f, 8f, 16f, 32f, 64f, 128f}}}, + {"Float32 dim 3, three rows (column-major slicing)", "QBit(Float32, 3)", 3, + "000000060707010000010000010000010000010000010006010701040401000201000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + false, new Object[] { + new float[] {1f, 2f, 3f}, new float[] {4f, 5f, 6f}, new float[] {7f, 8f, 9f}}}, + }; + } + + @Test(dataProvider = "qbitNativeGoldenBytes") + public void testReadQBitNativeDecodesGoldenBytes(String label, String columnType, int nRows, + String columnDataHex, boolean isDouble, Object[] expectedRows) throws Exception { + List rows = qbitReader(fromHex(columnDataHex)) + .readQBitNative(ClickHouseColumn.of("vec", columnType), nRows); + + Assert.assertEquals(rows.size(), nRows, label); + for (int r = 0; r < nRows; r++) { + Object vector = ((BinaryStreamReader.ArrayValue) rows.get(r)).getArray(); + if (isDouble) { + Assert.assertEquals((double[]) vector, (double[]) expectedRows[r], label + " row " + r); + } else { + Assert.assertEquals((float[]) vector, (float[]) expectedRows[r], label + " row " + r); + } + } + } + + @Test + public void testReadQBitNativeDecodesSpecialFloatValues() throws Exception { + // NaN, +Inf, -Inf and -0.0 must survive the bit-plane transpose bit-for-bit. + float[] vector = (float[]) ((BinaryStreamReader.ArrayValue) qbitReader(fromHex( + "0c07070707070707070100000000000000000000000000000000000000000000")) + .readQBitNative(ClickHouseColumn.of("vec", "QBit(Float32, 4)"), 1).get(0)).getArray(); + + float[] expected = {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, -0.0f}; + Assert.assertEquals(vector.length, expected.length); + for (int j = 0; j < expected.length; j++) { + Assert.assertEquals(Float.floatToRawIntBits(vector[j]), Float.floatToRawIntBits(expected[j]), + "element " + j); + } + } + + @Test + public void testReadQBitNativeRejectsUnsupportedElementType() { + // readQBitNative reconstructs only float element types; a non-float element is rejected by its own + // guard (the Native reader also filters these earlier), which fires before any read. + ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Int8, 3)"); + ClientException ex = Assert.expectThrows(ClientException.class, + () -> qbitReader(new byte[0]).readQBitNative(column, 1)); + Assert.assertTrue(ex.getMessage().contains("Float32"), + "Expected an unsupported-element-type message, got: " + ex.getMessage()); + } + + @Test + public void testReadQBitNativeRejectsIntOverflowPlaneSize() { + // nRows * ceil(dimension/8) per bit plane must not overflow a 32-bit int; the reader rejects it + // before allocating or reading, so an empty stream is sufficient (200000 -> 25000 bytes/plane, + // 90000 rows -> 2_250_000_000 bytes > Integer.MAX_VALUE). + ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Float32, 200000)"); + ClientException ex = Assert.expectThrows(ClientException.class, + () -> qbitReader(new byte[0]).readQBitNative(column, 90000)); + Assert.assertTrue(ex.getMessage().contains("too large"), + "Expected an overflow rejection message, got: " + ex.getMessage()); + } + + @Test + public void testDynamicDecimalKeepsPrecisionAndScaleInsideArray() throws Exception { + // 0x2x : the precision and the scale belong to the type, so a parent + // encoding (here Array) must keep them, otherwise the value is read back unscaled. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.Decimal64.getBinTag()); + BinaryStreamUtils.writeInt8(baos, 18); // precision + BinaryStreamUtils.writeInt8(baos, 4); // scale + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeInt64(baos, 12500); + BinaryStreamUtils.writeInt64(baos, -35000); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + Assert.assertEquals(values, new Object[]{new BigDecimal("1.2500"), new BigDecimal("-3.5000")}); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test + public void testDynamicEnum8KeepsConstantsAndReadsThemSigned() throws Exception { + // The constants belong to the type as well, and an Enum8 constant is a signed byte. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.Enum8.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeString(baos, "a'b"); + BinaryStreamUtils.writeInt8(baos, -1); + BinaryStreamUtils.writeString(baos, "c"); + BinaryStreamUtils.writeInt8(baos, 2); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeInt8(baos, -1); + BinaryStreamUtils.writeInt8(baos, 2); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + assertEnumValue(values[0], "a'b", -1); + assertEnumValue(values[1], "c", 2); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test + public void testDynamicEnum16TakesConstantWidthFromTag() throws Exception { + // The width of a constant is defined by the tag (Enum16 -> Int16), not by the number of + // constants: an Enum16 with less than 128 constants desynchronizes the stream otherwise. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.Enum16.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeString(baos, "a"); + BinaryStreamUtils.writeInt16(baos, -1000); + BinaryStreamUtils.writeString(baos, "b"); + BinaryStreamUtils.writeInt16(baos, 2000); + BinaryStreamUtils.writeVarInt(baos, 1); + BinaryStreamUtils.writeInt16(baos, -1000); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + assertEnumValue(values[0], "a", -1000); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test + public void testDynamicNestedConsumesElementTypeEncodings() throws Exception { + // Nested is encoded as a named tuple: every element name is followed by the type encoding + // of that element, which has to be consumed as well - including the parameters of that + // element type (here the precision and the scale of the decimal). + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Nested.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeString(baos, "a"); + baos.write(ClickHouseDataType.Decimal64.getBinTag()); + BinaryStreamUtils.writeInt8(baos, 18); // precision + BinaryStreamUtils.writeInt8(baos, 4); // scale + BinaryStreamUtils.writeString(baos, "b"); + baos.write(ClickHouseDataType.String.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 1); + BinaryStreamUtils.writeInt64(baos, 12500); + BinaryStreamUtils.writeString(baos, "x"); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] rows = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + Assert.assertEquals(rows.length, 1); + Assert.assertEquals((Object[]) rows[0], new Object[]{new BigDecimal("1.2500"), "x"}); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test + public void testDynamicVariantIsNotWrappedTwice() throws Exception { + // A variant rebuilt as Variant(Variant(...)) has a single element, so the discriminator + // selects the wrong alternative. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.Variant.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + baos.write(ClickHouseDataType.Int32.getBinTag()); + baos.write(ClickHouseDataType.String.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeInt8(baos, 1); // discriminator of String + BinaryStreamUtils.writeString(baos, "a"); + BinaryStreamUtils.writeInt8(baos, 0); // discriminator of Int32 + BinaryStreamUtils.writeInt32(baos, 1); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + Assert.assertEquals(values, new Object[]{"a", 1}); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + private static void assertEnumValue(Object actual, String expectedName, int expectedValue) { + BinaryStreamReader.EnumValue value = (BinaryStreamReader.EnumValue) actual; + Assert.assertEquals(value.getName(), expectedName); + Assert.assertEquals(value.intValue(), expectedValue); + } + + private static BinaryStreamReader dynamicReader(ByteArrayOutputStream columnData) { + return new BinaryStreamReader(new ByteArrayInputStream(columnData.toByteArray()), + TimeZone.getTimeZone("UTC"), null, new BinaryStreamReader.CachingByteBufferAllocator(), + false, null, false); + } + + private static BinaryStreamReader qbitReader(byte[] columnData) { + return new BinaryStreamReader(new ByteArrayInputStream(columnData), TimeZone.getTimeZone("UTC"), + null, new BinaryStreamReader.CachingByteBufferAllocator(), false, null, false); + } + + private static byte[] fromHex(String hex) { + byte[] out = new byte[hex.length() / 2]; + for (int i = 0; i < out.length; i++) { + out[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); + } + return out; + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index 045d69505..a29badb3a 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -1456,6 +1456,106 @@ public void testDynamicWithNestedTypes() throws Exception { Assert.assertEquals(records.get(0).getInteger("num"), 10); } + @DataProvider(name = "dynamicParametrizedElementTypes") + public Object[][] dynamicParametrizedElementTypes() { + return new Object[][]{ + {"Decimal32(2)", "[1.25]", new String[]{"1.25"}}, + {"Decimal64(4)", "[1.25, -3.5]", new String[]{"1.2500", "-3.5000"}}, + {"Decimal(10, 2)", "[12345678.91]", new String[]{"12345678.91"}}, + {"Decimal128(6)", "[0.000001]", new String[]{"0.000001"}}, + {"Decimal256(20)", "[1.5]", new String[]{"1.50000000000000000000"}}, + {"Enum8('a' = 1, 'b' = 2)", "['b']", new String[]{"b"}}, + {"Enum8('a' = -1, 'b' = 2)", "['a']", new String[]{"a"}}, + {"Enum8('a,b' = 1, 'c\\'d' = 2)", "['a,b', 'c\\'d']", new String[]{"a,b", "c'd"}}, + {"Enum16('a' = 1000, 'b' = 2000)", "['b']", new String[]{"b"}}, + {"Enum16('a' = -1000, 'b' = 2000)", "['a']", new String[]{"a"}}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "dynamicParametrizedElementTypes") + public void testDynamicWithParametrizedElementType(String elementType, String values, String[] expected) throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + List records = client.queryAll("SELECT " + values + "::Array(" + elementType + ")::Dynamic AS v, 42::Int32 AS num"); + Object[] items = records.get(0).getObjectArray("v"); + Assert.assertEquals(items.length, expected.length); + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals(String.valueOf(items[i]), expected[i]); + } + Assert.assertEquals(records.get(0).getInteger("num"), 42); + } + + @Test(groups = {"integration"}) + public void testDynamicWithEnum8WithMoreThan127Constants() throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + StringBuilder enumType = new StringBuilder("Enum8("); + for (int i = 0; i < 130; i++) { + if (i > 0) { + enumType.append(", "); + } + enumType.append('\'').append("c").append(i).append("' = ").append(i - 128); + } + enumType.append(')'); + + List records = client.queryAll("SELECT ['c0', 'c129']::Array(" + enumType + ")::Dynamic AS v, 42::Int32 AS num"); + Object[] items = records.get(0).getObjectArray("v"); + Assert.assertEquals(items.length, 2); + Assert.assertEquals(items[0].toString(), "c0"); + Assert.assertEquals(items[1].toString(), "c129"); + Assert.assertEquals(records.get(0).getInteger("num"), 42); + } + + @Test(groups = {"integration"}) + public void testDynamicWithParametrizedTypesInsideMapAndTuple() throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + List records = client.queryAll("SELECT " + + "map('k', 1.25::Decimal64(4))::Map(String, Decimal64(4))::Dynamic AS m, " + + "(1.25::Decimal64(4), 'b'::Enum8('a' = 1, 'b' = 2))::Tuple(d Decimal64(4), e Enum8('a' = 1, 'b' = 2))::Dynamic AS t, " + + "42::Int32 AS num"); + GenericRecord row = records.get(0); + Assert.assertEquals(String.valueOf(((Map) row.getObject("m")).get("k")), "1.2500"); + Object[] tuple = (Object[]) row.getObject("t"); + Assert.assertEquals(String.valueOf(tuple[0]), "1.2500"); + Assert.assertEquals(String.valueOf(tuple[1]), "b"); + Assert.assertEquals(row.getInteger("num"), 42); + } + + @Test(groups = {"integration"}) + public void testDynamicWithVariantElement() throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + // The variant elements are declared in an order that differs from the order the server encodes + // them in, so a wrongly rebuilt variant maps a discriminator to the wrong element. + List records = client.queryAll("SELECT [1, 'a']::Array(Variant(String, Int32))::Dynamic AS v, 42::Int32 AS num"); + Assert.assertEquals(records.get(0).getObjectArray("v"), new Object[]{1, "a"}); + Assert.assertEquals(records.get(0).getInteger("num"), 42); + } + + @Test(groups = {"integration"}) + public void testDynamicWithNestedElement() throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + List records = client.queryAll("SELECT [(1, 'x', 1.25), (2, 'y', -3.5)]" + + "::Nested(a Int32, b String, c Decimal64(4))::Dynamic AS v, 42::Int32 AS num"); + Object[] items = records.get(0).getObjectArray("v"); + Assert.assertEquals(items.length, 2); + Assert.assertEquals((Object[]) items[0], new Object[]{1, "x", new BigDecimal("1.2500")}); + Assert.assertEquals((Object[]) items[1], new Object[]{2, "y", new BigDecimal("-3.5000")}); + Assert.assertEquals(records.get(0).getInteger("num"), 42); + } + @Test(groups = {"integration"}) public void testDynamicWithFixedString() throws Exception { if (isVersionMatch("(,24.8]")) {