From f8c5577cf0d7bbe6a24863d3e54e0901707aeae0 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Wed, 4 Nov 2026 14:23:26 -0700 Subject: [PATCH 1/2] Chunk upload API Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Matt Klein --- .../protobuf/client/v1/api.proto | 75 +++++++++++++++++-- .../protobuf/client/v1/artifact.proto | 13 ++++ 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/bitdrift_public/protobuf/client/v1/api.proto b/src/bitdrift_public/protobuf/client/v1/api.proto index f01145f..6b6811f 100644 --- a/src/bitdrift_public/protobuf/client/v1/api.proto +++ b/src/bitdrift_public/protobuf/client/v1/api.proto @@ -429,13 +429,16 @@ enum ArtifactPayloadEncoding { } message UploadArtifactRequest { - // Upload UUID used to provide idempotence and to correlate a response with this request. + // Attempt UUID correlating the response. Whole-file uploads also use it for idempotence; + // chunked identity is authenticated owner + artifact_id, not this per-attempt value. string upload_uuid = 1 [(validate.rules).string = {min_len: 1}]; - // The type of the artifact being uploaded. - string type_id = 2 [(validate.rules).string = {min_len: 1}]; + // Required for whole-file uploads and chunked Begin/resume; thin chunk/Query leave this empty. + // The server enforces these operation-dependent requirements after ordinary PGV validation. + string type_id = 2; - // The artifact to upload. This is a binary blob that is interpreted by the server based on the type_id. + // The complete artifact for a whole-file upload, or the encoded chunk selected by chunk_index. + // Begin/resume and Query leave this empty; descriptors are not chunk payloads. bytes contents = 3; // A client-generated ID that uniquely identifies the artifact being uploaded. This is used to correlate @@ -446,8 +449,9 @@ message UploadArtifactRequest { // this may capture information about the device at the time of a crash. map state_metadata = 5; - // The timestamp associated with this upload. This allows us to possibly reject the upload of very old artifacts. - google.protobuf.Timestamp time = 6 [(validate.rules).message = {required: true}]; + // Capture timestamp, required for whole-file uploads and chunked Begin/resume. Thin chunk/Query + // omit it; on non-data chunked requests its presence selects Begin/resume rather than Query. + google.protobuf.Timestamp time = 6; // The session ID associated with the artifact if applicable. string session_id = 7; @@ -463,7 +467,8 @@ message UploadArtifactRequest { // this field; workflow command artifacts remain session artifacts and leave it unset. optional string command_id = 10 [(validate.rules).string.uuid = true]; - // The encoding of contents. Workflow/direct command attachments and crash profiles may use zlib. + // Whole-file payload encoding. Workflow/direct command attachments and crash profiles may use zlib. + // Chunked uploads leave this RAW; their descriptors are the sole source of chunk encoding. ArtifactPayloadEncoding payload_encoding = 11; // The MIME type of a workflow attachment, direct command attachment, or crash profile. @@ -474,14 +479,68 @@ message UploadArtifactRequest { // Report links for type_id = "crash_profile". CrashProfileMetadata crash_profile = 13; + + // Presence selects chunked upload, including a zero-length remote file. Available for config + // version >= 52 with runtime admission enabled; server support is assumed, not negotiated. + // Every chunk-mode request retains this mode/size marker, but full metadata is sent only at + // Begin/resume. Thin chunk/Query use authenticated, durably admitted server metadata. + // IDs/MIME are canonical; origin is bounded; whole-file-only fields stay empty. + optional uint64 logical_bytes = 14 [(validate.rules).uint64.lte = 268435456]; + bytes logical_sha256 = 15 [(validate.rules).bytes = { + ignore_empty: true + len: 32 + }]; + // Descriptors for the entire logical file, not multiple payloads or concurrent uploads. + // Only Begin/resume sends this list; chunk/Query leave it empty. + repeated ArtifactChunkDescriptor chunks = 16 [(validate.rules).repeated.max_items = 256]; + // Present, including zero: upload exactly one chunk using the server's admitted descriptor, + // omitting manifest-only fields. Absent: Begin/resume with full metadata, or thin Query. + // logical_bytes stays present; + // time presence selects Begin/resume, and its absence selects Query with manifest fields empty. + // All durable receipts automatically make the upload eligible for durable finalization. + // Abandonment does not cancel the command or extend its deadline. + optional uint32 chunk_index = 17 [(validate.rules).uint32.lt = 256]; } message UploadArtifactResponse { + enum State { + // Whole-file ACKs leave state unset. This is never a successful chunked ACK. + STATE_UNSPECIFIED = 0; + STATE_RECEIVING = 1; + STATE_FINALIZING = 2; + // Canonical object, authoritative catalog/receipt and required processing notification are + // durable. Processing success is separate. Only this state completes a chunked artifact. + STATE_FINALIZED = 3; + // Back off and repeat Begin/resume with the original durable metadata and retained bytes. + STATE_RETRYABLE = 4; + // Permanent rejection/retirement. The SDK may release retained bytes and report failure. + STATE_REJECTED = 5; + } + // The UUID corresponding to the upload request. string upload_uuid = 1 [(validate.rules).string = {min_len: 1}]; - // Optional error message which indicates that artifact upload failed. + // Whole-file error semantics are unchanged. Chunked retryable/rejected states require an error of + // at most 1024 bytes and omit the bitmap; active/finalized states require an empty error. string error = 2; + + // Chunked progress only. Whole-file ACK semantics and fields 1-2 are unchanged. + State state = 3 [(validate.rules).enum.defined_only = true]; + // Exactly ceil(admitted chunk count / 8) bytes, low bit first. Unused high bits are zero. + // Bits reflect authoritative durable receipts, not object listings or attempted transmissions. + bytes received_chunks = 4 [(validate.rules).bytes.max_len = 32]; +} + +// Fixed 1 MiB logical chunks, with a short last chunk. Position in the admitted manifest determines +// index, offset and logical size. ZLIB is permitted only when it is smaller than the logical bytes. +message ArtifactChunkDescriptor { + uint32 encoded_bytes = 1 [(validate.rules).uint32 = { + gt: 0 + lte: 1048576 + }]; + ArtifactPayloadEncoding encoding = 2 [(validate.rules).enum.defined_only = true]; + bytes logical_sha256 = 3 [(validate.rules).bytes.len = 32]; + bytes encoded_sha256 = 4 [(validate.rules).bytes.len = 32]; } // The response sent as part of stream establishment. diff --git a/src/bitdrift_public/protobuf/client/v1/artifact.proto b/src/bitdrift_public/protobuf/client/v1/artifact.proto index ce2e564..63e982a 100644 --- a/src/bitdrift_public/protobuf/client/v1/artifact.proto +++ b/src/bitdrift_public/protobuf/client/v1/artifact.proto @@ -47,6 +47,15 @@ enum StorageFormat { // the upload manager to coordinate intents and uploads. message ArtifactUploadIndex { + // Chunk ownership uses the same index commit/recovery as whole-file artifacts. A separate entry + // field prevents an older SDK from interpreting a chunk or manifest as a complete artifact payload. + message ChunkedArtifact { + // Payload-free Begin/resume template: no attempt UUID, contents or current chunk index. + // The planned M5 uploader retains this full manifest locally but derives thin chunk/Query + // requests from identity and logical size, adding only attempt UUID and chunk index/bytes. + UploadArtifactRequest request = 1; + } + message Artifact { // Name of the file to upload. string name = 1; @@ -93,6 +102,10 @@ message ArtifactUploadIndex { // List of files, in order of time, that are pending upload. repeated Artifact artifact = 1; + + // M1 defines metadata only. M2 wires shared atomic/fsynced commits, recovery and payload cleanup. + // Retain all chunks until finalized or explicit rejection. + repeated ChunkedArtifact chunked_artifact = 2; } // Durable local ownership and retirement state for a workflow attachment. From f7391e91286db380398539d0a76723fca7797098 Mon Sep 17 00:00:00 2001 From: Matt Klein Date: Mon, 16 Nov 2026 11:52:33 -0700 Subject: [PATCH 2/2] simplify/docs Signed-off-by: Matt Klein --- .../protobuf/client/v1/api.proto | 60 ++++++++++++------- .../protobuf/client/v1/artifact.proto | 18 ++---- 2 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/bitdrift_public/protobuf/client/v1/api.proto b/src/bitdrift_public/protobuf/client/v1/api.proto index 6b6811f..8b70d24 100644 --- a/src/bitdrift_public/protobuf/client/v1/api.proto +++ b/src/bitdrift_public/protobuf/client/v1/api.proto @@ -428,17 +428,33 @@ enum ArtifactPayloadEncoding { ARTIFACT_PAYLOAD_ENCODING_ZLIB = 1; } +// Uploads an artifact either as one whole-file request or as a resumable sequence of chunks. +// Chunked uploads require configuration version >= 52 and runtime admission to be enabled. +// +// A chunked upload has two operations: +// 1. Begin/resume: omit chunk_index and contents, and send the complete immutable metadata, +// logical file size/checksum and ordered chunk descriptors. Repeating Begin reconciles progress +// after a disconnect or retry; it must describe the same artifact as the original Begin. +// 2. PutChunk: set chunk_index and contents, and send only upload_uuid, artifact_id, command_id +// (if applicable) and logical_bytes alongside them. The server uses the admitted descriptor +// to decode and verify this chunk. +// +// Responses identify durably stored chunks with a receipt bitmap. Send missing chunks, retaining +// all local payloads until STATE_FINALIZED or STATE_REJECTED. Once all chunks are durable, the server +// publishes the artifact and any required processing notification without a separate finalize RPC. +// On retry or unfinished publication, back off and repeat Begin with the original metadata. +// Finalization does not imply processing success. Stopping uploads does not cancel a command or +// extend its deadline. message UploadArtifactRequest { // Attempt UUID correlating the response. Whole-file uploads also use it for idempotence; // chunked identity is authenticated owner + artifact_id, not this per-attempt value. string upload_uuid = 1 [(validate.rules).string = {min_len: 1}]; - // Required for whole-file uploads and chunked Begin/resume; thin chunk/Query leave this empty. - // The server enforces these operation-dependent requirements after ordinary PGV validation. + // Artifact type. Required for whole-file uploads and Begin/resume; omitted on PutChunk. string type_id = 2; // The complete artifact for a whole-file upload, or the encoded chunk selected by chunk_index. - // Begin/resume and Query leave this empty; descriptors are not chunk payloads. + // Begin/resume leaves this empty. bytes contents = 3; // A client-generated ID that uniquely identifies the artifact being uploaded. This is used to correlate @@ -449,8 +465,7 @@ message UploadArtifactRequest { // this may capture information about the device at the time of a crash. map state_metadata = 5; - // Capture timestamp, required for whole-file uploads and chunked Begin/resume. Thin chunk/Query - // omit it; on non-data chunked requests its presence selects Begin/resume rather than Query. + // Capture timestamp. Required for whole-file uploads and Begin/resume; omitted on PutChunk. google.protobuf.Timestamp time = 6; // The session ID associated with the artifact if applicable. @@ -480,25 +495,20 @@ message UploadArtifactRequest { // Report links for type_id = "crash_profile". CrashProfileMetadata crash_profile = 13; - // Presence selects chunked upload, including a zero-length remote file. Available for config - // version >= 52 with runtime admission enabled; server support is assumed, not negotiated. - // Every chunk-mode request retains this mode/size marker, but full metadata is sent only at - // Begin/resume. Thin chunk/Query use authenticated, durably admitted server metadata. - // IDs/MIME are canonical; origin is bounded; whole-file-only fields stay empty. + // Total uncompressed file size. Presence selects chunked upload, including a zero-length remote + // command file with no chunks. Required and unchanged on every operation for this artifact. optional uint64 logical_bytes = 14 [(validate.rules).uint64.lte = 268435456]; + // SHA-256 of the entire uncompressed file, in chunk order. Required on Begin/resume and omitted + // on PutChunk. This identifies the logical file independently of its transfer/storage encoding. bytes logical_sha256 = 15 [(validate.rules).bytes = { ignore_empty: true len: 32 }]; - // Descriptors for the entire logical file, not multiple payloads or concurrent uploads. - // Only Begin/resume sends this list; chunk/Query leave it empty. + // Ordered descriptors covering the entire logical file. Required on Begin/resume except for + // an empty file; omitted on PutChunk. Each descriptor describes one separately uploaded chunk. repeated ArtifactChunkDescriptor chunks = 16 [(validate.rules).repeated.max_items = 256]; - // Present, including zero: upload exactly one chunk using the server's admitted descriptor, - // omitting manifest-only fields. Absent: Begin/resume with full metadata, or thin Query. - // logical_bytes stays present; - // time presence selects Begin/resume, and its absence selects Query with manifest fields empty. - // All durable receipts automatically make the upload eligible for durable finalization. - // Abandonment does not cancel the command or extend its deadline. + // Presence selects PutChunk, including index zero. Absence selects Begin/resume, which must carry + // the complete metadata and descriptors described above. optional uint32 chunk_index = 17 [(validate.rules).uint32.lt = 256]; } @@ -506,15 +516,15 @@ message UploadArtifactResponse { enum State { // Whole-file ACKs leave state unset. This is never a successful chunked ACK. STATE_UNSPECIFIED = 0; + // The upload is not finalized. received_chunks describes its durable payloads. STATE_RECEIVING = 1; - STATE_FINALIZING = 2; // Canonical object, authoritative catalog/receipt and required processing notification are // durable. Processing success is separate. Only this state completes a chunked artifact. - STATE_FINALIZED = 3; + STATE_FINALIZED = 2; // Back off and repeat Begin/resume with the original durable metadata and retained bytes. - STATE_RETRYABLE = 4; + STATE_RETRYABLE = 3; // Permanent rejection/retirement. The SDK may release retained bytes and report failure. - STATE_REJECTED = 5; + STATE_REJECTED = 4; } // The UUID corresponding to the upload request. @@ -534,12 +544,18 @@ message UploadArtifactResponse { // Fixed 1 MiB logical chunks, with a short last chunk. Position in the admitted manifest determines // index, offset and logical size. ZLIB is permitted only when it is smaller than the logical bytes. message ArtifactChunkDescriptor { + // Exact number of bytes sent in this chunk's contents, before server-side decoding. uint32 encoded_bytes = 1 [(validate.rules).uint32 = { gt: 0 lte: 1048576 }]; + // Encoding of the transmitted bytes, independent of how the server stores the chunk. ArtifactPayloadEncoding encoding = 2 [(validate.rules).enum.defined_only = true]; + // SHA-256 after decoding. Used to verify logical content, including after server recompression. bytes logical_sha256 = 3 [(validate.rules).bytes.len = 32]; + // SHA-256 before decoding. Allows the SDK to verify retained encoded files cheaply before + // transmission and the server to reject changed encoded bytes before decompression. + // For RAW chunks this is identical to logical_sha256. bytes encoded_sha256 = 4 [(validate.rules).bytes.len = 32]; } diff --git a/src/bitdrift_public/protobuf/client/v1/artifact.proto b/src/bitdrift_public/protobuf/client/v1/artifact.proto index 63e982a..32a495d 100644 --- a/src/bitdrift_public/protobuf/client/v1/artifact.proto +++ b/src/bitdrift_public/protobuf/client/v1/artifact.proto @@ -47,15 +47,6 @@ enum StorageFormat { // the upload manager to coordinate intents and uploads. message ArtifactUploadIndex { - // Chunk ownership uses the same index commit/recovery as whole-file artifacts. A separate entry - // field prevents an older SDK from interpreting a chunk or manifest as a complete artifact payload. - message ChunkedArtifact { - // Payload-free Begin/resume template: no attempt UUID, contents or current chunk index. - // The planned M5 uploader retains this full manifest locally but derives thin chunk/Query - // requests from identity and logical size, adding only attempt UUID and chunk index/bytes. - UploadArtifactRequest request = 1; - } - message Artifact { // Name of the file to upload. string name = 1; @@ -103,9 +94,12 @@ message ArtifactUploadIndex { // List of files, in order of time, that are pending upload. repeated Artifact artifact = 1; - // M1 defines metadata only. M2 wires shared atomic/fsynced commits, recovery and payload cleanup. - // Retain all chunks until finalized or explicit rejection. - repeated ChunkedArtifact chunked_artifact = 2; + // Chunked uploads pending finalization, committed atomically with whole-file ownership above. + // Each entry is a complete, payload-free Begin/resume template: upload_uuid, contents and + // chunk_index are unset. Network attempts add a fresh upload_uuid; PutChunk requests derive their + // identity and descriptor from this template. Retain all chunk files until finalized or rejected. + // The separate field keeps older SDKs from interpreting chunked entries as whole-file payloads. + repeated UploadArtifactRequest chunked_artifact = 2; } // Durable local ownership and retirement state for a workflow attachment.