Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 83 additions & 8 deletions src/bitdrift_public/protobuf/client/v1/api.proto
Original file line number Diff line number Diff line change
Expand Up @@ -428,14 +428,33 @@ enum ArtifactPayloadEncoding {
ARTIFACT_PAYLOAD_ENCODING_ZLIB = 1;
}

// Uploads an artifact either as one whole-file request or as a resumable sequence of chunks.
// Chunked uploads require configuration version >= 52 and runtime admission to be enabled.
//
// A chunked upload has two operations:
// 1. Begin/resume: omit chunk_index and contents, and send the complete immutable metadata,
// logical file size/checksum and ordered chunk descriptors. Repeating Begin reconciles progress
// after a disconnect or retry; it must describe the same artifact as the original Begin.
// 2. PutChunk: set chunk_index and contents, and send only upload_uuid, artifact_id, command_id
// (if applicable) and logical_bytes alongside them. The server uses the admitted descriptor
// to decode and verify this chunk.
//
// Responses identify durably stored chunks with a receipt bitmap. Send missing chunks, retaining
// all local payloads until STATE_FINALIZED or STATE_REJECTED. Once all chunks are durable, the server
// publishes the artifact and any required processing notification without a separate finalize RPC.
// On retry or unfinished publication, back off and repeat Begin with the original metadata.
// Finalization does not imply processing success. Stopping uploads does not cancel a command or
// extend its deadline.
message UploadArtifactRequest {
// Upload UUID used to provide idempotence and to correlate a response with this request.
// Attempt UUID correlating the response. Whole-file uploads also use it for idempotence;
// chunked identity is authenticated owner + artifact_id, not this per-attempt value.
string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

// The type of the artifact being uploaded.
string type_id = 2 [(validate.rules).string = {min_len: 1}];
// Artifact type. Required for whole-file uploads and Begin/resume; omitted on PutChunk.
string type_id = 2;

// The artifact to upload. This is a binary blob that is interpreted by the server based on the type_id.
// The complete artifact for a whole-file upload, or the encoded chunk selected by chunk_index.
// Begin/resume leaves this empty.
bytes contents = 3;

// A client-generated ID that uniquely identifies the artifact being uploaded. This is used to correlate
Expand All @@ -446,8 +465,8 @@ message UploadArtifactRequest {
// this may capture information about the device at the time of a crash.
map<string, logging.v1.Data> state_metadata = 5;

// The timestamp associated with this upload. This allows us to possibly reject the upload of very old artifacts.
google.protobuf.Timestamp time = 6 [(validate.rules).message = {required: true}];
// Capture timestamp. Required for whole-file uploads and Begin/resume; omitted on PutChunk.
google.protobuf.Timestamp time = 6;

// The session ID associated with the artifact if applicable.
string session_id = 7;
Expand All @@ -463,7 +482,8 @@ message UploadArtifactRequest {
// this field; workflow command artifacts remain session artifacts and leave it unset.
optional string command_id = 10 [(validate.rules).string.uuid = true];

// The encoding of contents. Workflow/direct command attachments and crash profiles may use zlib.
// Whole-file payload encoding. Workflow/direct command attachments and crash profiles may use zlib.
// Chunked uploads leave this RAW; their descriptors are the sole source of chunk encoding.
ArtifactPayloadEncoding payload_encoding = 11;

// The MIME type of a workflow attachment, direct command attachment, or crash profile.
Expand All @@ -474,14 +494,69 @@ message UploadArtifactRequest {

// Report links for type_id = "crash_profile".
CrashProfileMetadata crash_profile = 13;

// Total uncompressed file size. Presence selects chunked upload, including a zero-length remote
// command file with no chunks. Required and unchanged on every operation for this artifact.
optional uint64 logical_bytes = 14 [(validate.rules).uint64.lte = 268435456];
// SHA-256 of the entire uncompressed file, in chunk order. Required on Begin/resume and omitted
// on PutChunk. This identifies the logical file independently of its transfer/storage encoding.
bytes logical_sha256 = 15 [(validate.rules).bytes = {
ignore_empty: true
len: 32
}];
// Ordered descriptors covering the entire logical file. Required on Begin/resume except for
// an empty file; omitted on PutChunk. Each descriptor describes one separately uploaded chunk.
repeated ArtifactChunkDescriptor chunks = 16 [(validate.rules).repeated.max_items = 256];
// Presence selects PutChunk, including index zero. Absence selects Begin/resume, which must carry
// the complete metadata and descriptors described above.
optional uint32 chunk_index = 17 [(validate.rules).uint32.lt = 256];
}

message UploadArtifactResponse {
enum State {
// Whole-file ACKs leave state unset. This is never a successful chunked ACK.
STATE_UNSPECIFIED = 0;
// The upload is not finalized. received_chunks describes its durable payloads.
STATE_RECEIVING = 1;
// Canonical object, authoritative catalog/receipt and required processing notification are
// durable. Processing success is separate. Only this state completes a chunked artifact.
STATE_FINALIZED = 2;
// Back off and repeat Begin/resume with the original durable metadata and retained bytes.
STATE_RETRYABLE = 3;
// Permanent rejection/retirement. The SDK may release retained bytes and report failure.
STATE_REJECTED = 4;
}

// The UUID corresponding to the upload request.
string upload_uuid = 1 [(validate.rules).string = {min_len: 1}];

// Optional error message which indicates that artifact upload failed.
// Whole-file error semantics are unchanged. Chunked retryable/rejected states require an error of
// at most 1024 bytes and omit the bitmap; active/finalized states require an empty error.
string error = 2;

// Chunked progress only. Whole-file ACK semantics and fields 1-2 are unchanged.
State state = 3 [(validate.rules).enum.defined_only = true];
// Exactly ceil(admitted chunk count / 8) bytes, low bit first. Unused high bits are zero.
// Bits reflect authoritative durable receipts, not object listings or attempted transmissions.
bytes received_chunks = 4 [(validate.rules).bytes.max_len = 32];
}

// Fixed 1 MiB logical chunks, with a short last chunk. Position in the admitted manifest determines
// index, offset and logical size. ZLIB is permitted only when it is smaller than the logical bytes.
Comment thread
mattklein123 marked this conversation as resolved.
Resolved
message ArtifactChunkDescriptor {
// Exact number of bytes sent in this chunk's contents, before server-side decoding.
uint32 encoded_bytes = 1 [(validate.rules).uint32 = {
gt: 0
lte: 1048576
}];
// Encoding of the transmitted bytes, independent of how the server stores the chunk.
ArtifactPayloadEncoding encoding = 2 [(validate.rules).enum.defined_only = true];
// SHA-256 after decoding. Used to verify logical content, including after server recompression.
bytes logical_sha256 = 3 [(validate.rules).bytes.len = 32];
// SHA-256 before decoding. Allows the SDK to verify retained encoded files cheaply before
// transmission and the server to reject changed encoded bytes before decompression.
// For RAW chunks this is identical to logical_sha256.
bytes encoded_sha256 = 4 [(validate.rules).bytes.len = 32];
Comment thread
mattklein123 marked this conversation as resolved.
Resolved
}

// The response sent as part of stream establishment.
Expand Down
7 changes: 7 additions & 0 deletions src/bitdrift_public/protobuf/client/v1/artifact.proto
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,13 @@ message ArtifactUploadIndex {

// List of files, in order of time, that are pending upload.
repeated Artifact artifact = 1;

// Chunked uploads pending finalization, committed atomically with whole-file ownership above.
// Each entry is a complete, payload-free Begin/resume template: upload_uuid, contents and
// chunk_index are unset. Network attempts add a fresh upload_uuid; PutChunk requests derive their
// identity and descriptor from this template. Retain all chunk files until finalized or rejected.
// The separate field keeps older SDKs from interpreting chunked entries as whole-file payloads.
repeated UploadArtifactRequest chunked_artifact = 2;
}

// Durable local ownership and retirement state for a workflow attachment.
Expand Down
Loading