diff --git a/src/bitdrift_public/protobuf/client/v1/api.proto b/src/bitdrift_public/protobuf/client/v1/api.proto index f01145f..8b70d24 100644 --- a/src/bitdrift_public/protobuf/client/v1/api.proto +++ b/src/bitdrift_public/protobuf/client/v1/api.proto @@ -428,14 +428,33 @@ enum ArtifactPayloadEncoding { ARTIFACT_PAYLOAD_ENCODING_ZLIB = 1; } +// Uploads an artifact either as one whole-file request or as a resumable sequence of chunks. +// Chunked uploads require configuration version >= 52 and runtime admission to be enabled. +// +// A chunked upload has two operations: +// 1. Begin/resume: omit chunk_index and contents, and send the complete immutable metadata, +// logical file size/checksum and ordered chunk descriptors. Repeating Begin reconciles progress +// after a disconnect or retry; it must describe the same artifact as the original Begin. +// 2. PutChunk: set chunk_index and contents, and send only upload_uuid, artifact_id, command_id +// (if applicable) and logical_bytes alongside them. The server uses the admitted descriptor +// to decode and verify this chunk. +// +// Responses identify durably stored chunks with a receipt bitmap. Send missing chunks, retaining +// all local payloads until STATE_FINALIZED or STATE_REJECTED. Once all chunks are durable, the server +// publishes the artifact and any required processing notification without a separate finalize RPC. +// On retry or unfinished publication, back off and repeat Begin with the original metadata. +// Finalization does not imply processing success. Stopping uploads does not cancel a command or +// extend its deadline. message UploadArtifactRequest { - // Upload UUID used to provide idempotence and to correlate a response with this request. + // Attempt UUID correlating the response. Whole-file uploads also use it for idempotence; + // chunked identity is authenticated owner + artifact_id, not this per-attempt value. string upload_uuid = 1 [(validate.rules).string = {min_len: 1}]; - // The type of the artifact being uploaded. - string type_id = 2 [(validate.rules).string = {min_len: 1}]; + // Artifact type. Required for whole-file uploads and Begin/resume; omitted on PutChunk. + string type_id = 2; - // The artifact to upload. This is a binary blob that is interpreted by the server based on the type_id. + // The complete artifact for a whole-file upload, or the encoded chunk selected by chunk_index. + // Begin/resume leaves this empty. bytes contents = 3; // A client-generated ID that uniquely identifies the artifact being uploaded. This is used to correlate @@ -446,8 +465,8 @@ message UploadArtifactRequest { // this may capture information about the device at the time of a crash. map state_metadata = 5; - // The timestamp associated with this upload. This allows us to possibly reject the upload of very old artifacts. - google.protobuf.Timestamp time = 6 [(validate.rules).message = {required: true}]; + // Capture timestamp. Required for whole-file uploads and Begin/resume; omitted on PutChunk. + google.protobuf.Timestamp time = 6; // The session ID associated with the artifact if applicable. string session_id = 7; @@ -463,7 +482,8 @@ message UploadArtifactRequest { // this field; workflow command artifacts remain session artifacts and leave it unset. optional string command_id = 10 [(validate.rules).string.uuid = true]; - // The encoding of contents. Workflow/direct command attachments and crash profiles may use zlib. + // Whole-file payload encoding. Workflow/direct command attachments and crash profiles may use zlib. + // Chunked uploads leave this RAW; their descriptors are the sole source of chunk encoding. ArtifactPayloadEncoding payload_encoding = 11; // The MIME type of a workflow attachment, direct command attachment, or crash profile. @@ -474,14 +494,69 @@ message UploadArtifactRequest { // Report links for type_id = "crash_profile". CrashProfileMetadata crash_profile = 13; + + // Total uncompressed file size. Presence selects chunked upload, including a zero-length remote + // command file with no chunks. Required and unchanged on every operation for this artifact. + optional uint64 logical_bytes = 14 [(validate.rules).uint64.lte = 268435456]; + // SHA-256 of the entire uncompressed file, in chunk order. Required on Begin/resume and omitted + // on PutChunk. This identifies the logical file independently of its transfer/storage encoding. + bytes logical_sha256 = 15 [(validate.rules).bytes = { + ignore_empty: true + len: 32 + }]; + // Ordered descriptors covering the entire logical file. Required on Begin/resume except for + // an empty file; omitted on PutChunk. Each descriptor describes one separately uploaded chunk. + repeated ArtifactChunkDescriptor chunks = 16 [(validate.rules).repeated.max_items = 256]; + // Presence selects PutChunk, including index zero. Absence selects Begin/resume, which must carry + // the complete metadata and descriptors described above. + optional uint32 chunk_index = 17 [(validate.rules).uint32.lt = 256]; } message UploadArtifactResponse { + enum State { + // Whole-file ACKs leave state unset. This is never a successful chunked ACK. + STATE_UNSPECIFIED = 0; + // The upload is not finalized. received_chunks describes its durable payloads. + STATE_RECEIVING = 1; + // Canonical object, authoritative catalog/receipt and required processing notification are + // durable. Processing success is separate. Only this state completes a chunked artifact. + STATE_FINALIZED = 2; + // Back off and repeat Begin/resume with the original durable metadata and retained bytes. + STATE_RETRYABLE = 3; + // Permanent rejection/retirement. The SDK may release retained bytes and report failure. + STATE_REJECTED = 4; + } + // The UUID corresponding to the upload request. string upload_uuid = 1 [(validate.rules).string = {min_len: 1}]; - // Optional error message which indicates that artifact upload failed. + // Whole-file error semantics are unchanged. Chunked retryable/rejected states require an error of + // at most 1024 bytes and omit the bitmap; active/finalized states require an empty error. string error = 2; + + // Chunked progress only. Whole-file ACK semantics and fields 1-2 are unchanged. + State state = 3 [(validate.rules).enum.defined_only = true]; + // Exactly ceil(admitted chunk count / 8) bytes, low bit first. Unused high bits are zero. + // Bits reflect authoritative durable receipts, not object listings or attempted transmissions. + bytes received_chunks = 4 [(validate.rules).bytes.max_len = 32]; +} + +// Fixed 1 MiB logical chunks, with a short last chunk. Position in the admitted manifest determines +// index, offset and logical size. ZLIB is permitted only when it is smaller than the logical bytes. +message ArtifactChunkDescriptor { + // Exact number of bytes sent in this chunk's contents, before server-side decoding. + uint32 encoded_bytes = 1 [(validate.rules).uint32 = { + gt: 0 + lte: 1048576 + }]; + // Encoding of the transmitted bytes, independent of how the server stores the chunk. + ArtifactPayloadEncoding encoding = 2 [(validate.rules).enum.defined_only = true]; + // SHA-256 after decoding. Used to verify logical content, including after server recompression. + bytes logical_sha256 = 3 [(validate.rules).bytes.len = 32]; + // SHA-256 before decoding. Allows the SDK to verify retained encoded files cheaply before + // transmission and the server to reject changed encoded bytes before decompression. + // For RAW chunks this is identical to logical_sha256. + bytes encoded_sha256 = 4 [(validate.rules).bytes.len = 32]; } // The response sent as part of stream establishment. diff --git a/src/bitdrift_public/protobuf/client/v1/artifact.proto b/src/bitdrift_public/protobuf/client/v1/artifact.proto index ce2e564..32a495d 100644 --- a/src/bitdrift_public/protobuf/client/v1/artifact.proto +++ b/src/bitdrift_public/protobuf/client/v1/artifact.proto @@ -93,6 +93,13 @@ message ArtifactUploadIndex { // List of files, in order of time, that are pending upload. repeated Artifact artifact = 1; + + // Chunked uploads pending finalization, committed atomically with whole-file ownership above. + // Each entry is a complete, payload-free Begin/resume template: upload_uuid, contents and + // chunk_index are unset. Network attempts add a fresh upload_uuid; PutChunk requests derive their + // identity and descriptor from this template. Retain all chunk files until finalized or rejected. + // The separate field keeps older SDKs from interpreting chunked entries as whole-file payloads. + repeated UploadArtifactRequest chunked_artifact = 2; } // Durable local ownership and retirement state for a workflow attachment.