diff --git a/specification/src/main/asciidoc/chapters/Sparkplug_6_Payloads.adoc b/specification/src/main/asciidoc/chapters/Sparkplug_6_Payloads.adoc index 682fb29e..9fed1e33 100644 --- a/specification/src/main/asciidoc/chapters/Sparkplug_6_Payloads.adoc +++ b/specification/src/main/asciidoc/chapters/Sparkplug_6_Payloads.adoc @@ -221,6 +221,7 @@ message Payload { Template template_value = 13; PropertySet propertyset_value = 14; PropertySetList propertysets_value = 15; + ValueArray array_value = 16; } } @@ -251,19 +252,31 @@ message Payload { Status status = 4; // Execution status - present in NRESP/DRESP responses, omitted in BIRTH definitions and CMD requests } + // An array of scalar values of one type, used both by the array DataTypes and by + // DataSet.Column. At most one of the typed arrays is populated - none for an empty array - + // chosen by the datatype of the enclosing Metric or Column. The signed arrays use zigzag + // encoding because a negative plain varint occupies ten bytes. + message ValueArray { + repeated sint32 sint32_values = 1; // Int8, Int16, Int32 + repeated sint64 sint64_values = 2; // Int64 + repeated uint32 uint32_values = 3; // UInt8, UInt16, UInt32 + repeated uint64 uint64_values = 4; // UInt64 + repeated int64 int64_values = 5; // DateTime; not zigzag - pre-1970 costs 10 bytes + repeated float float_values = 6; // Float + repeated double double_values = 7; // Double + repeated bool boolean_values = 8; // Boolean + repeated string string_values = 9; // String, Text + + // The indexes of the elements that are null. Omitted entirely when none are. + repeated uint32 null_indexes = 10; + } + message DataSet { message Column { - string name = 1; - DataType datatype = 2; - - // Only one type of repeated value shall be used per Column - repeated uint32 int_values = 3; - repeated uint64 long_values = 4; - repeated float float_values = 5; - repeated double double_values = 6; - repeated bool boolean_values = 7; - repeated string string_values = 8; + string name = 1; + DataType datatype = 2; + ValueArray values = 3; // one entry per row, in row order } repeated Column columns = 1; @@ -1057,55 +1070,21 @@ Column MUST include a datatype, which is the datatype of every value in that col datatype MUST be one of the enumerated values shown in the link:#payloads_c_datatype_basic[Sparkplug Basic Data Types].*# * *values* -** The values of a Column are carried in one of six repeated fields, selected by the Column's -datatype as shown in the table below. -** [tck-testable tck-id-payloads-dataset-column-values-field]#[yellow-background]*[tck-id-payloads-dataset-column-values-field] The -values of a Column MUST be carried in the repeated field that corresponds to that Column's datatype -as shown in the DataSet Column Value Fields table.*# -** [tck-testable tck-id-payloads-dataset-column-values-one]#[yellow-background]*[tck-id-payloads-dataset-column-values-one] A -Column MUST NOT populate more than one of the six repeated value fields.*# -** [tck-testable tck-id-payloads-dataset-column-values-signed]#[yellow-background]*[tck-id-payloads-dataset-column-values-signed] A -signed value carried in int_values or long_values MUST be encoded as its two's complement -representation, and a consuming application MUST interpret it according to the Column's datatype.*# - -The following table defines which repeated field carries the values of a Column for each datatype. - -.DataSet Column Value Fields -[cols="2,2,4"] -|=== -|*Column datatype* -|*Field* -|*Google Protocol Buffers Type* - -|Int8, Int16, Int32, UInt8, UInt16, UInt32 -|int_values -|repeated uint32, packed - -|Int64, UInt64, DateTime -|long_values -|repeated uint64, packed - -|Float -|float_values -|repeated float, packed - -|Double -|double_values -|repeated double, packed - -|Boolean -|boolean_values -|repeated bool, packed - -|String, Text -|string_values -|repeated string, not packed -|=== - -Non-normative comment: There is no signed repeated field, so the signed datatypes share int_values -and long_values with their unsigned counterparts. A publishing application writes the value's two's -complement bit pattern and a consuming application narrows it back to the declared datatype, which -is the same convention the Metric value fields use for Int8 and Int16. +** The values of a Column are carried in a ValueArray, one entry per row, in row order. The typed +array within it is selected by the Column's datatype, as described in the +link:#payloads_c_datatype_array[Array Types] section, which defines the same structure for the +array DataTypes. +** [tck-testable tck-id-payloads-dataset-column-values-field]#[yellow-background]*[tck-id-payloads-dataset-column-values-field] A +DataSet.Column MUST carry its values in a ValueArray, populating only the typed array that +corresponds to that Column's datatype.*# +** [tck-testable tck-id-payloads-dataset-column-null]#[yellow-background]*[tck-id-payloads-dataset-column-null] A +null cell MUST be recorded by its row index in that ValueArray's null_indexes field, so that the +column keeps one entry per row.*# +** Non-normative comment: a Sparkplug 3 DataSet was row oriented and a null cell was an empty +DataSetValue, which held its position with no value set. A column oriented DataSet has no +equivalent, because a repeated field is dense, so null_indexes restores what the row oriented form +could express. Without it an encoder would have to omit the cell, which shortens the column and +shifts every later row, or write a zero, which is indistinguishable from a real zero. [[payloads_c_dataset_example]] ===== DataSet Example @@ -1121,15 +1100,15 @@ two rows. The first row is (1, 72.5, true) and the second is (2, 80.0, false). "columns": [{ "name": "Step", "datatype": "UInt32", - "int_values": [1, 2] + "values": { "uint32_values": [1, 2] } }, { "name": "Setpoint", "datatype": "Double", - "double_values": [72.5, 80.0] + "values": { "double_values": [72.5, 80.0] } }, { "name": "Enabled", "datatype": "Boolean", - "boolean_values": [true, false] + "values": { "boolean_values": [true, false] } }] } } @@ -1747,9 +1726,9 @@ enum DataType { ** Non-normative comment: a DateTime value is a nanosecond count since epoch, the same quantity the payload and metric timestamps carry, and is signed for the same reason. Every mainstream time representation it interoperates with is signed, so the ceiling is 2262-04-11 as described in the -link:#operational_behavior_timestamps[Timestamps] section. A DateTime carried in a DataSet.Column -shares the long_values array with UInt64, as the -link:#payloads_c_dataset_column[DataSet.Column] section describes. +link:#operational_behavior_timestamps[Timestamps] section. A DateTime carried in an array or a DataSet.Column +uses the int64_values array of the ValueArray, as the +link:#payloads_c_datatype_array[Array Types] section describes. * _Text_ ** String value (UTF-8) ** Google Protocol Buffer Type: string @@ -1792,84 +1771,77 @@ link:#payloads_c_dataset_column[DataSet.Column] section describes. [[payloads_c_datatype_array]] * *Array Types* -All array types use the bytes_value field of the Metric value field. They are simply little-endian -packed byte arrays. - -For example, consider an Int32 array with two decimal values [123456789, 987654321] - -Array converted to little endian hex: [0x15CD5B07, 0xB168DE3A] - -The bytes_value of the Sparkplug Metric must be: [0x15, 0xCD, 0x5B, 0x07, 0xB1, 0x68, 0xDE, 0x3A] +An array DataType carries its elements in a ValueArray, in the array_value field of the Metric's +TypedValue. At most one of the ValueArray's typed arrays is populated, selected by the datatype as +listed below, and the elements appear in order; an array with no elements populates none. + +** [tck-testable tck-id-payloads-array-value-array]#[yellow-background]*[tck-id-payloads-array-value-array] A +Metric whose datatype is an array type MUST carry its elements in the array_value field of its +TypedValue.*# +** [tck-testable tck-id-payloads-array-single-field]#[yellow-background]*[tck-id-payloads-array-single-field] A +ValueArray MUST populate at most one typed array, which MUST be the one that corresponds to the +datatype of the enclosing Metric or DataSet.Column. An array with no elements populates none.*# +** [tck-testable tck-id-payloads-array-null-indexes]#[yellow-background]*[tck-id-payloads-array-null-indexes] An +element that is null MUST be recorded by its index in the null_indexes field, and the typed array +MUST still hold a value at that index so that every element keeps its position.*# +** [tck-testable tck-id-payloads-array-null-indexes-range]#[yellow-background]*[tck-id-payloads-array-null-indexes-range] Every +value in null_indexes MUST be a valid index of the populated typed array, MUST NOT appear more than +once, and the indexes MUST appear in ascending order.*# +** [tck-not-testable tck-id-payloads-array-null-value-ignored]#[yellow-background]*[tck-id-payloads-array-null-value-ignored] A +consuming application MUST treat an element whose index appears in null_indexes as null, whatever +value the typed array holds at that index.*# +** Non-normative comment: the signed arrays use zigzag encoding because a negative value in a plain +varint field occupies ten bytes, which would make a signed array substantially larger than the +packed byte array this replaces. DateTime is the exception: its values are large positive nanosecond +counts, for which zigzag only adds a bit, so it uses a plain varint array. +** Non-normative comment: null_indexes exists because a repeated field is dense and has no +per-element presence. Recording the indexes rather than carrying a parallel array of flags costs +nothing when an array has no nulls, which is the ordinary case, and a few bytes per null otherwise. +A publisher still writes some value at a null element's position, and a consuming application +ignores it. +** For example, an Int32Array of [123456789, 987654321] populates sint32_values with those two +elements. An Int32Array of [10, null, 30] populates sint32_values with three elements and sets +null_indexes to [1]. ** _Int8Array_ -*** Int8Array as an array of packed little endian int8 bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the sint32_values field of the ValueArray *** Sparkplug enum value: 22 -*** Example (Decimal to Metric bytes_value): [-23, 123] -> [0xE9, 0x7B] ** _Int16Array_ -*** Int16Array as an array of packed little endian int16 bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the sint32_values field of the ValueArray *** Sparkplug enum value: 23 -*** Example (Decimal to Metric bytes_value): [-30000, 30000] -> [0xD0, 0x8A, 0x30, 0x75] ** _Int32Array_ -*** Int8Array as an array of packed little endian int32 bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the sint32_values field of the ValueArray *** Sparkplug enum value: 24 -*** Example (Decimal to Metric bytes_value): [-1, 315338746] -> [0xFF, 0xFF, 0xFF, 0xFF, 0xFA, 0xAF, 0xCB, 0x12] ** _Int64Array_ -*** Int8Array as an array of packed little endian int64 bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the sint64_values field of the ValueArray *** Sparkplug enum value: 25 -*** Example (Decimal to Metric bytes_value): [-4270929666821191986, -3601064768563266876] -> [0xCE, 0x06, 0x72, 0xAC, 0x18, 0x9C, 0xBA, 0xC4, 0xC4, 0xBA, 0x9C, 0x18, 0xAC, 0x72, 0x06, 0xCE] ** _UInt8Array_ -*** UInt8Array as an array of packed little endian uint8 bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the uint32_values field of the ValueArray *** Sparkplug enum value: 26 -*** Example (Decimal to Metric bytes_value): [23, 250] -> [0x17, 0xFA] ** _UInt16Array_ -*** UInt16Array as an array of packed little endian uint16 bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the uint32_values field of the ValueArray *** Sparkplug enum value: 27 -*** Example (Decimal to Metric bytes_value): [30, 52360] -> [0x1E, 0x00, 0x88, 0xCC] ** _UInt32Array_ -*** UInt32Array as an array of packed little endian uint32 bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the uint32_values field of the ValueArray *** Sparkplug enum value: 28 -*** Example (Decimal to Metric bytes_value): [52, 3293969225] -> [0x34, 0x00, 0x00, 0x00, 0x49, 0xFB, 0x55, 0xC4] ** _UInt64Array_ -*** UInt64Array as an array of packed little endian uint64 bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the uint64_values field of the ValueArray *** Sparkplug enum value: 29 -*** Example (Decimal to Metric bytes_value): [52, 16444743074749521625] -> [0x34, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xD9, 0x9E, 0x02, 0xD1, 0xB2, 0x76, 0x37, 0xE4] ** _FloatArray_ -*** FloatArray as an array of packed little endian 32-bit float bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the float_values field of the ValueArray *** Sparkplug enum value: 30 -*** Example (Decimal to Metric bytes_value): [1.23, 89.341] -> [0xA4, 0x70, 0x9D, 0x3F, 0x98, 0xAE, 0xB2, 0x42] ** _DoubleArray_ -*** DoubleArray as an array of packed little endian 64-bit float bytes -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the double_values field of the ValueArray *** Sparkplug enum value: 31 -*** Example (Decimal to Metric bytes_value): [12.354213, 1022.9123213] -> [0xD7, 0xA2, 0x05, 0x68, 0x5B, 0xB5, 0x28, 0x40, 0x8E, 0x17, 0x1C, 0x6F, 0x4C, 0xF7, 0x8F, 0x40] ** _BooleanArray_ -*** BooleanArray as an array of bit-packed bytes preceeded by a 4-byte integer that represents the -total number of boolean values -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the boolean_values field of the ValueArray *** Sparkplug enum value: 32 -*** Example (boolean array to Metric bytes_value): [false, false, true, true, false, true, false, false, true, true, false, true] -> [0x0C, 0x00, 0x00, 0x00, 0x34, 0xDX] -**** Note an 'X' above is a 'do not care'. It can be either 1 or 0 but must be present so the array -ends on a byte boundary. ** _StringArray_ -*** StringArray as an array of null terminated strings -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the string_values field of the ValueArray *** Sparkplug enum value: 33 -*** Example (string array to Metric bytes_value): [ABC, hello] -> [0x41, 0x42, 0x43, 0x00, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x00] ** _DateTimeArray_ -*** DateTimeArray as an array of packed little endian bytes where each DateTime value is a signed -8-byte value representing the number of nanoseconds since epoch in UTC -*** Google Protocol Buffer Type: bytes +*** Elements are carried in the int64_values field of the ValueArray *** Sparkplug enum value: 34 -*** Example (DateTime array -> nanoseconds since epoch -> Metric bytes_value): ['Wednesday, October 21, 2009 5:27:55.335 AM', 'Friday, June 24, 2022 9:57:55 PM'] -> [1256102875335000000, 1656107875000000000] -> [0xC0, 0x7F, 0xB0, 0xF5, 0xE8, 0x92, 0x6E, 0x11, 0x00, 0x1E, 0x1A, 0x7F, 0x56, 0xAD, 0xFB, 0x16] [[payloads_c_datatype_command]] * *Command Type* @@ -3791,13 +3763,19 @@ because it is observable in a payload, whereas the absence of meaning in the ord [[payloads_json_array_types]] ===== Array Types -In the Protocol Buffers encoding the array DataTypes are carried as packed little-endian bytes, as -described in the link:#payloads_c_datatype_array[Array Types] section. Carrying those same packed -bytes in JSON would make an array value opaque to a JSON consumer. +In the Protocol Buffers encoding the array DataTypes and a DataSet.Column carry their elements in a +ValueArray, which selects one of several typed arrays and records absent elements separately, as +described in the link:#payloads_c_datatype_array[Array Types] section. In JSON the datatype is +already known and a JSON array can hold a null directly, so a ValueArray is represented as a plain +array. [tck-testable tck-id-payloads-json-array-values]#[yellow-background]*[tck-id-payloads-json-array-values] A -value of an array DataType MUST be represented as a JSON array of the element values, not as a -base64 encoded packed byte array.*# +ValueArray MUST be represented as a JSON array of its element values, rather than as an object +carrying one of the typed arrays named in the Google Protocol Buffers schema.*# + +[tck-testable tck-id-payloads-json-array-null]#[yellow-background]*[tck-id-payloads-json-array-null] An +element whose index appears in null_indexes MUST be represented as a JSON null, and the JSON array +MUST NOT carry a separate null_indexes key.*# [tck-testable tck-id-payloads-json-array-element-type]#[yellow-background]*[tck-id-payloads-json-array-element-type] Each element of that JSON array MUST be represented according to the scalar mapping for the element @@ -3807,14 +3785,14 @@ of an Int32Array are JSON numbers.*# [[payloads_json_dataset]] ===== DataSet -In the Protocol Buffers schema a DataSet.Column carries its values in one of six parallel arrays, -one per underlying Protocol Buffers type, of which exactly one is used. In JSON the column's -datatype already identifies the element type. +In the Protocol Buffers schema a DataSet.Column carries its values in a ValueArray, which selects +a typed array by the column's datatype and records absent cells separately. In JSON the datatype is +already known and a JSON array can hold a null directly. [tck-testable tck-id-payloads-json-dataset-values]#[yellow-background]*[tck-id-payloads-json-dataset-values] A -DataSet.Column MUST carry its values as a single JSON array under the key 'values', rather than as -one of the typed value arrays named in the Google Protocol Buffers schema, and each element MUST be -represented according to the scalar mapping for the column's datatype.*# +DataSet.Column MUST carry its values as a single JSON array under the key 'values', one entry per +row, with each element represented according to the scalar mapping for the column's datatype and a +null cell represented as a JSON null.*# [[payloads_json_content]] ===== Content and Compression @@ -3903,11 +3881,19 @@ Sparkplug JSON payload MUST validate against the JSON Schema below.*# "body": { "$ref": "#/$defs/base64" } } }, + "arrayElement": { + "comment": "an element of an array DataType or DataSet.Column; null marks an absent element", + "anyOf": [ + { "type": "null" }, + { "type": [ "string", "boolean" ] }, + { "$ref": "#/$defs/number" } + ] + }, "value": { "anyOf": [ { "type": [ "string", "boolean" ] }, { "$ref": "#/$defs/number" }, - { "type": "array" }, + { "type": "array", "items": { "$ref": "#/$defs/arrayElement" } }, { "$ref": "#/$defs/dataset" }, { "$ref": "#/$defs/template" }, { "$ref": "#/$defs/propertyset" }, @@ -3970,7 +3956,7 @@ Sparkplug JSON payload MUST validate against the JSON Schema below.*# "properties": { "name": { "type": "string" }, "datatype": { "$ref": "#/$defs/datatype" }, - "values": { "type": "array" } + "values": { "type": "array", "items": { "$ref": "#/$defs/arrayElement" } } } } } diff --git a/tck/src/main/java/org/eclipse/sparkplug/impl/exception/message/SparkplugBPayloadEncoder.java b/tck/src/main/java/org/eclipse/sparkplug/impl/exception/message/SparkplugBPayloadEncoder.java index 354c5a81..a1157a5c 100644 --- a/tck/src/main/java/org/eclipse/sparkplug/impl/exception/message/SparkplugBPayloadEncoder.java +++ b/tck/src/main/java/org/eclipse/sparkplug/impl/exception/message/SparkplugBPayloadEncoder.java @@ -439,166 +439,201 @@ private SparkplugBProto.Payload.Metric.Builder setMetricValue(SparkplugBProto.Pa // Add the template to the metric metricBuilder.getValueBuilder().setTemplateValue(templateBuilder); break; - case Int8Array: - Byte[] int8ArrayValue = (Byte[]) metric.getValue(); - ByteBuffer int8ByteBuffer = - ByteBuffer.allocate(int8ArrayValue.length).order(ByteOrder.LITTLE_ENDIAN); - for (Byte value : int8ArrayValue) { - int8ByteBuffer.put(value); - } - if (int8ByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(int8ByteBuffer.array())); + case Int8Array: { + SparkplugBProto.Payload.ValueArray.Builder vaInt8Array = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iInt8Array = 0; + for (Byte element : (Byte[]) metric.getValue()) { + if (element == null) { + vaInt8Array.addNullIndexes(iInt8Array); + vaInt8Array.addSint32Values(0); + } else { + vaInt8Array.addSint32Values(element); + } + iInt8Array++; } break; - case Int16Array: - Short[] int16ArrayValue = (Short[]) metric.getValue(); - ByteBuffer int16ByteBuffer = - ByteBuffer.allocate(int16ArrayValue.length * 2).order(ByteOrder.LITTLE_ENDIAN); - for (Short value : int16ArrayValue) { - int16ByteBuffer.putShort(value); - } - if (int16ByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(int16ByteBuffer.array())); + } + case Int16Array: { + SparkplugBProto.Payload.ValueArray.Builder vaInt16Array = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iInt16Array = 0; + for (Short element : (Short[]) metric.getValue()) { + if (element == null) { + vaInt16Array.addNullIndexes(iInt16Array); + vaInt16Array.addSint32Values(0); + } else { + vaInt16Array.addSint32Values(element); + } + iInt16Array++; } break; - case Int32Array: - Integer[] int32ArrayValue = (Integer[]) metric.getValue(); - ByteBuffer int32ByteBuffer = - ByteBuffer.allocate(int32ArrayValue.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (Integer value : int32ArrayValue) { - int32ByteBuffer.putInt(value); - } - if (int32ByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(int32ByteBuffer.array())); + } + case Int32Array: { + SparkplugBProto.Payload.ValueArray.Builder vaInt32Array = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iInt32Array = 0; + for (Integer element : (Integer[]) metric.getValue()) { + if (element == null) { + vaInt32Array.addNullIndexes(iInt32Array); + vaInt32Array.addSint32Values(0); + } else { + vaInt32Array.addSint32Values(element); + } + iInt32Array++; } break; - case Int64Array: - Long[] int64ArrayValue = (Long[]) metric.getValue(); - ByteBuffer int64ByteBuffer = - ByteBuffer.allocate(int64ArrayValue.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (Long value : int64ArrayValue) { - int64ByteBuffer.putLong(value); - } - if (int64ByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(int64ByteBuffer.array())); + } + case Int64Array: { + SparkplugBProto.Payload.ValueArray.Builder vaInt64Array = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iInt64Array = 0; + for (Long element : (Long[]) metric.getValue()) { + if (element == null) { + vaInt64Array.addNullIndexes(iInt64Array); + vaInt64Array.addSint64Values(0L); + } else { + vaInt64Array.addSint64Values(element); + } + iInt64Array++; } break; - case UInt8Array: - Short[] uInt8ArrayValue = (Short[]) metric.getValue(); - ByteBuffer uInt8ByteBuffer = - ByteBuffer.allocate(uInt8ArrayValue.length).order(ByteOrder.LITTLE_ENDIAN); - for (Short value : uInt8ArrayValue) { - uInt8ByteBuffer.put((byte) (value & 0xffff)); - } - if (uInt8ByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(uInt8ByteBuffer.array())); + } + case UInt8Array: { + SparkplugBProto.Payload.ValueArray.Builder vaUInt8Array = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iUInt8Array = 0; + for (Short element : (Short[]) metric.getValue()) { + if (element == null) { + vaUInt8Array.addNullIndexes(iUInt8Array); + vaUInt8Array.addUint32Values(0); + } else { + vaUInt8Array.addUint32Values(Short.toUnsignedInt(element)); + } + iUInt8Array++; } break; - case UInt16Array: - Integer[] uInt16ArrayValue = (Integer[]) metric.getValue(); - ByteBuffer uInt16ByteBuffer = - ByteBuffer.allocate(uInt16ArrayValue.length * 2).order(ByteOrder.LITTLE_ENDIAN); - for (Integer value : uInt16ArrayValue) { - uInt16ByteBuffer.putShort((short) (value & 0xffffffff)); - } - if (uInt16ByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(uInt16ByteBuffer.array())); + } + case UInt16Array: { + SparkplugBProto.Payload.ValueArray.Builder vaUInt16Array = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iUInt16Array = 0; + for (Integer element : (Integer[]) metric.getValue()) { + if (element == null) { + vaUInt16Array.addNullIndexes(iUInt16Array); + vaUInt16Array.addUint32Values(0); + } else { + vaUInt16Array.addUint32Values((int) Integer.toUnsignedLong(element)); + } + iUInt16Array++; } break; - case UInt32Array: - Long[] uInt32ArrayValue = (Long[]) metric.getValue(); - ByteBuffer uInt32ByteBuffer = - ByteBuffer.allocate(uInt32ArrayValue.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (Long value : uInt32ArrayValue) { - uInt32ByteBuffer.putInt((int) (value & 0xffffffffffffffffL)); - } - if (uInt32ByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(uInt32ByteBuffer.array())); + } + case UInt32Array: { + SparkplugBProto.Payload.ValueArray.Builder vaUInt32Array = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iUInt32Array = 0; + for (Long element : (Long[]) metric.getValue()) { + if (element == null) { + vaUInt32Array.addNullIndexes(iUInt32Array); + vaUInt32Array.addUint32Values(0); + } else { + vaUInt32Array.addUint32Values(element.intValue()); + } + iUInt32Array++; } break; - case UInt64Array: - BigInteger[] uInt64ArrayValue = (BigInteger[]) metric.getValue(); - ByteBuffer uInt64ByteBuffer = - ByteBuffer.allocate(uInt64ArrayValue.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (BigInteger value : uInt64ArrayValue) { - uInt64ByteBuffer.putLong(bigIntegerToUnsignedLong(value)); - } - if (uInt64ByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(uInt64ByteBuffer.array())); + } + case UInt64Array: { + SparkplugBProto.Payload.ValueArray.Builder vaUInt64Array = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iUInt64Array = 0; + for (BigInteger element : (BigInteger[]) metric.getValue()) { + if (element == null) { + vaUInt64Array.addNullIndexes(iUInt64Array); + vaUInt64Array.addUint64Values(0L); + } else { + vaUInt64Array.addUint64Values(bigIntegerToUnsignedLong(element)); + } + iUInt64Array++; } break; - case FloatArray: - Float[] floatArrayValue = (Float[]) metric.getValue(); - ByteBuffer floatByteBuffer = - ByteBuffer.allocate(floatArrayValue.length * 4).order(ByteOrder.LITTLE_ENDIAN); - for (Float value : floatArrayValue) { - floatByteBuffer.putFloat(value); - } - if (floatByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(floatByteBuffer.array())); + } + case FloatArray: { + SparkplugBProto.Payload.ValueArray.Builder vaFloatArray = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iFloatArray = 0; + for (Float element : (Float[]) metric.getValue()) { + if (element == null) { + vaFloatArray.addNullIndexes(iFloatArray); + vaFloatArray.addFloatValues(0f); + } else { + vaFloatArray.addFloatValues(element); + } + iFloatArray++; } break; - case DoubleArray: - Double[] doubleArrayValue = (Double[]) metric.getValue(); - ByteBuffer doubleByteBuffer = - ByteBuffer.allocate(doubleArrayValue.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (Double value : doubleArrayValue) { - doubleByteBuffer.putDouble(value); - } - if (doubleByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(doubleByteBuffer.array())); + } + case DoubleArray: { + SparkplugBProto.Payload.ValueArray.Builder vaDoubleArray = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iDoubleArray = 0; + for (Double element : (Double[]) metric.getValue()) { + if (element == null) { + vaDoubleArray.addNullIndexes(iDoubleArray); + vaDoubleArray.addDoubleValues(0d); + } else { + vaDoubleArray.addDoubleValues(element); + } + iDoubleArray++; } break; - case BooleanArray: - Boolean[] booleanArrayValue = (Boolean[]) metric.getValue(); - int numberOfBytes = (int) Math.ceil((double) booleanArrayValue.length / 8); - ByteBuffer booleanByteBuffer = - ByteBuffer.allocate(4 + numberOfBytes).order(ByteOrder.LITTLE_ENDIAN); - - // The first 4 bytes is the number of booleans in the array - booleanByteBuffer.putInt(booleanArrayValue.length); - - // Get the remaining bytes - for (int i = 0; i < numberOfBytes; i++) { - byte nextByte = 0; - for (int bit = 0; bit < 8; bit++) { - int index = i * 8 + bit; - if (index < booleanArrayValue.length && booleanArrayValue[index]) { - nextByte |= (128 >> bit); - } + } + case BooleanArray: { + SparkplugBProto.Payload.ValueArray.Builder vaBooleanArray = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iBooleanArray = 0; + for (Boolean element : (Boolean[]) metric.getValue()) { + if (element == null) { + vaBooleanArray.addNullIndexes(iBooleanArray); + vaBooleanArray.addBooleanValues(false); + } else { + vaBooleanArray.addBooleanValues(element); } - booleanByteBuffer.put(nextByte); + iBooleanArray++; } - - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(booleanByteBuffer.array())); break; - case StringArray: - String[] stringArrayValue = (String[]) metric.getValue(); - int size = 0; - for (String string : stringArrayValue) { - size = size + string.length() + 1; - } - ByteBuffer stringByteBuffer = ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); - for (String value : stringArrayValue) { - byte[] stringBytes = value.getBytes(StandardCharsets.UTF_8); - stringByteBuffer.put(stringBytes); - stringByteBuffer.put((byte) 0); - } - if (stringByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(stringByteBuffer.array())); + } + case StringArray: { + SparkplugBProto.Payload.ValueArray.Builder vaStringArray = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iStringArray = 0; + for (String element : (String[]) metric.getValue()) { + if (element == null) { + vaStringArray.addNullIndexes(iStringArray); + vaStringArray.addStringValues(""); + } else { + vaStringArray.addStringValues(element); + } + iStringArray++; } break; - case DateTimeArray: - Date[] dateTimeArrayValue = (Date[]) metric.getValue(); - ByteBuffer dateTimeByteBuffer = - ByteBuffer.allocate(dateTimeArrayValue.length * 8).order(ByteOrder.LITTLE_ENDIAN); - for (Date value : dateTimeArrayValue) { - dateTimeByteBuffer.putLong(value.getTime()); - } - if (dateTimeByteBuffer.hasArray()) { - metricBuilder.getValueBuilder().setBytesValue(ByteString.copyFrom(dateTimeByteBuffer.array())); + } + case DateTimeArray: { + SparkplugBProto.Payload.ValueArray.Builder vaDateTimeArray = + metricBuilder.getValueBuilder().getArrayValueBuilder(); + int iDateTimeArray = 0; + for (Date element : (Date[]) metric.getValue()) { + if (element == null) { + vaDateTimeArray.addNullIndexes(iDateTimeArray); + vaDateTimeArray.addInt64Values(0L); + } else { + vaDateTimeArray.addInt64Values(element.getTime()); + } + iDateTimeArray++; } break; + } case Unknown: default: logger.error("Unsupported MetricDataType: " + metric.getDataType()); @@ -650,97 +685,136 @@ private SparkplugBProto.Payload.Metric.Builder setMetaData(SparkplugBProto.Paylo // so a value is appended to that array rather than wrapped in a per-cell message. private void addDataSetValue(SparkplugBProto.Payload.DataSet.Column.Builder columnBuilder, DataSetDataType type, Value value) throws Exception { + SparkplugBProto.Payload.ValueArray.Builder va = columnBuilder.getValuesBuilder(); Object raw = value == null ? null : value.getValue(); if (raw == null) { + // Record the index as null, then write a placeholder so the column keeps one entry + // per row. A consuming application ignores the placeholder. + va.addNullIndexes(nextRowIndex(va, type)); addDataSetPlaceholder(columnBuilder, type); return; } switch (type) { case Int8: - columnBuilder.addIntValues((Byte) raw); + va.addSint32Values((Byte) raw); break; case Int16: - columnBuilder.addIntValues((Short) raw); + va.addSint32Values((Short) raw); break; case Int32: - columnBuilder.addIntValues((Integer) raw); + va.addSint32Values((Integer) raw); break; case UInt8: - columnBuilder.addIntValues(Short.toUnsignedInt((Short) raw)); + va.addUint32Values(Short.toUnsignedInt((Short) raw)); break; case UInt16: - columnBuilder.addIntValues((int) Integer.toUnsignedLong((Integer) raw)); + va.addUint32Values((int) Integer.toUnsignedLong((Integer) raw)); break; case UInt32: if (raw instanceof Long) { - columnBuilder.addIntValues(((Long) raw).intValue()); + va.addUint32Values(((Long) raw).intValue()); } else { - columnBuilder.addIntValues((Integer) raw); + va.addUint32Values((Integer) raw); } break; case Int64: - columnBuilder.addLongValues((Long) raw); + va.addSint64Values((Long) raw); break; case UInt64: if (raw instanceof BigInteger) { - columnBuilder.addLongValues(bigIntegerToUnsignedLong((BigInteger) raw)); + va.addUint64Values(bigIntegerToUnsignedLong((BigInteger) raw)); } else { - columnBuilder.addLongValues((Long) raw); + va.addUint64Values((Long) raw); } break; case DateTime: - columnBuilder.addLongValues(((Date) raw).getTime()); + va.addInt64Values(((Date) raw).getTime()); break; case Float: - columnBuilder.addFloatValues((Float) raw); + va.addFloatValues((Float) raw); break; case Double: - columnBuilder.addDoubleValues((Double) raw); + va.addDoubleValues((Double) raw); break; case Boolean: - columnBuilder.addBooleanValues(toBoolean(raw)); + va.addBooleanValues(toBoolean(raw)); break; case String: case Text: - columnBuilder.addStringValues((String) raw); + va.addStringValues((String) raw); break; default: throw new Exception("Unknown DataSet column type: " + type); } } - // A v4 DataSet.Column carries a parallel array indexed by row and cannot express an absent - // cell, so a null holds its position with the zero value for the column's datatype. Every - // Column therefore ends with the same number of values. + // The index the next appended element will occupy in the array this datatype uses. + private int nextRowIndex(SparkplugBProto.Payload.ValueArray.Builder va, DataSetDataType type) { + switch (type) { + case Int8: + case Int16: + case Int32: + return va.getSint32ValuesCount(); + case UInt8: + case UInt16: + case UInt32: + return va.getUint32ValuesCount(); + case Int64: + return va.getSint64ValuesCount(); + case UInt64: + return va.getUint64ValuesCount(); + case DateTime: + return va.getInt64ValuesCount(); + case Float: + return va.getFloatValuesCount(); + case Double: + return va.getDoubleValuesCount(); + case Boolean: + return va.getBooleanValuesCount(); + default: + return va.getStringValuesCount(); + } + } + + // Writes the placeholder for a null cell, into the same typed array the column's datatype + // uses, so the column keeps one entry per row. The zero itself carries no meaning: the cell + // is recorded as null in null_indexes, and a consumer ignores the placeholder. private void addDataSetPlaceholder(SparkplugBProto.Payload.DataSet.Column.Builder columnBuilder, DataSetDataType type) throws Exception { + SparkplugBProto.Payload.ValueArray.Builder va = columnBuilder.getValuesBuilder(); switch (type) { case Int8: case Int16: case Int32: + va.addSint32Values(0); + break; case UInt8: case UInt16: case UInt32: - columnBuilder.addIntValues(0); + va.addUint32Values(0); break; case Int64: + va.addSint64Values(0L); + break; case UInt64: + va.addUint64Values(0L); + break; case DateTime: - columnBuilder.addLongValues(0L); + va.addInt64Values(0L); break; case Float: - columnBuilder.addFloatValues(0f); + va.addFloatValues(0f); break; case Double: - columnBuilder.addDoubleValues(0d); + va.addDoubleValues(0d); break; case Boolean: - columnBuilder.addBooleanValues(false); + va.addBooleanValues(false); break; case String: case Text: - columnBuilder.addStringValues(""); + va.addStringValues(""); break; default: throw new Exception("Unknown DataSet column type: " + type);