diff --git a/Cargo.lock b/Cargo.lock index 730b71ff..f4d2eb6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "c0mpute-api" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "c0mpute-proto", @@ -501,7 +501,7 @@ dependencies = [ [[package]] name = "c0mpute-bench" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "blake3", @@ -512,7 +512,7 @@ dependencies = [ [[package]] name = "c0mpute-cli" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "axum", @@ -542,7 +542,7 @@ dependencies = [ [[package]] name = "c0mpute-core" -version = "0.3.2" +version = "0.3.3" dependencies = [ "aes-gcm", "anyhow", @@ -585,12 +585,13 @@ dependencies = [ "url", "uuid", "x25519-dalek", + "zeroize", "zip", ] [[package]] name = "c0mpute-doctor" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "serde", @@ -602,7 +603,7 @@ dependencies = [ [[package]] name = "c0mpute-envelope" -version = "0.3.2" +version = "0.3.3" dependencies = [ "blake3", "bs58", @@ -615,7 +616,7 @@ dependencies = [ [[package]] name = "c0mpute-gateway" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "axum", @@ -637,7 +638,7 @@ dependencies = [ [[package]] name = "c0mpute-market" -version = "0.3.2" +version = "0.3.3" dependencies = [ "c0mpute-envelope", "serde", @@ -647,7 +648,7 @@ dependencies = [ [[package]] name = "c0mpute-net" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "async-trait", @@ -660,11 +661,12 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "zeroize", ] [[package]] name = "c0mpute-placement" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "async-trait", @@ -685,7 +687,7 @@ dependencies = [ [[package]] name = "c0mpute-proto" -version = "0.3.2" +version = "0.3.3" dependencies = [ "blake3", "hex", @@ -697,7 +699,7 @@ dependencies = [ [[package]] name = "c0mpute-sealed" -version = "0.3.2" +version = "0.3.3" dependencies = [ "aes", "aes-gcm", @@ -719,7 +721,7 @@ dependencies = [ [[package]] name = "c0mpute-secure-chat" -version = "0.3.2" +version = "0.3.3" dependencies = [ "aes-gcm", "anyhow", @@ -742,7 +744,7 @@ dependencies = [ [[package]] name = "c0mpute-store" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "blake3", @@ -760,7 +762,7 @@ dependencies = [ [[package]] name = "c0mpute-transcode" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "c0mpute-proto", @@ -773,7 +775,7 @@ dependencies = [ [[package]] name = "c0mpute-update" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "hex", @@ -788,7 +790,7 @@ dependencies = [ [[package]] name = "c0mpute-verify" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "blake3", diff --git a/Cargo.toml b/Cargo.toml index 86795190..9681f70d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ members = [ ] [workspace.package] -version = "0.3.2" +version = "0.3.3" edition = "2024" license = "MIT" repository = "https://github.com/profullstack/c0mpute" diff --git a/apps/tui/package.json b/apps/tui/package.json index 06895112..06487052 100644 --- a/apps/tui/package.json +++ b/apps/tui/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/tui", - "version": "0.3.2", + "version": "0.3.3", "private": true, "type": "module", "bin": { diff --git a/apps/web/package.json b/apps/web/package.json index 1076a466..67201874 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute", - "version": "0.3.2", + "version": "0.3.3", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/app/releases/latest.json/route.ts b/apps/web/src/app/releases/latest.json/route.ts index c1f5614a..4d00d044 100644 --- a/apps/web/src/app/releases/latest.json/route.ts +++ b/apps/web/src/app/releases/latest.json/route.ts @@ -28,7 +28,7 @@ export const dynamic = "force-dynamic"; * this in sync with the workspace version in `Cargo.toml` on each release so a * freshly built CLI reports "already latest" rather than a phantom upgrade. */ -const FALLBACK_VERSION = "0.3.2"; +const FALLBACK_VERSION = "0.3.3"; const FALLBACK_MIN_REQUIRED = "0.0.1"; type Channel = "stable" | "beta" | "nightly"; diff --git a/dips/0032-e2e-encrypted-jobs.md b/dips/0032-e2e-encrypted-jobs.md new file mode 100644 index 00000000..c3d85c80 --- /dev/null +++ b/dips/0032-e2e-encrypted-jobs.md @@ -0,0 +1,183 @@ +--- +dip: 0032 +title: "End-to-end encrypted jobs: only the winning worker sees a job, only the buyer sees its result" +status: Accepted +authors: + - riotcoder@profullstack.com +created: 2026-11-09 +updated: 2026-11-09 +discussion: +implementation: +--- + +# DIP 0032: End-to-end encrypted jobs + +## What + +Every job on the network (chat and inference, TTS, STT, image, music and +video generation, embeddings, OCR, transcode, render, crawl, `workflow.step`, +text training and inference, temporal, webhook, gameserver, captcha) is now +end-to-end encrypted between the buyer and the one worker that wins its +auction: + +- the offer that goes to gossip carries only what bidding needs; +- the job itself is sealed to the winner and handed over point to point; +- inputs too big for the job (audio, images, documents, corpora, adapters, + `.blend` files) are stored as ciphertext under a per-input key that only + the winner learns; +- the result is sealed back to the buyer before the worker stores it. + +## Why + +Job topics (`c0mpute/jobs/`) are gossipsub topics. Every peer and +relay subscribed to a topic received every offer on it, and until this DIP an +offer carried the whole job inline: every prompt, every transcript, every +URL, to every node on the topic, not just the one that won. Inputs and +results were plaintext chunks that anyone holding the hash (which the offer +and the receipt published) could fetch. Node operators had access to +people's data. That kept sensitive work (CoinPay's finance AI, for one) off +the network. + +## The protocol + +```text +buyer -> gossip offer e2e: { v: 1, buyer_key: B, public_spec } + spec_inline: absent + spec_hash: salted commitment to the spec + required_capabilities: [..., "c0mpute:e2e:v1"] +worker -> buyer bid e2e: { worker_key: W, sig: ed25519(identity, binding) } + (direct and on gossip, as before) +buyer -> both accept names the winner, e2e: 1, no spec +winner -> buyer rpc c0mpute.job.payload.v1 (request-response) + the buyer answers only the authenticated winner + with the spec sealed to W +winner -> store result sealed to the buyer; the receipt names the + ciphertext's hash +``` + +### Keys + +- `B` is a fresh X25519 key per auction; `W` a fresh X25519 key per bid. + Neither is ever reused, so a later compromise of either side opens nothing + old. +- The worker signs the binding of `W` to the job with its libp2p ed25519 + identity: `lp(domain) lp("bid") lp(job_id) lp(workload) lp(buyer_peer) + lp(worker_peer) lp(B) lp(W)` (every field length-prefixed). The buyer + verifies it against the bidder's peer id, which inlines the ed25519 key, + before it seals anything to `W`. A bid without a valid signature is + ignored. +- `shared = X25519(B, W)`, refused if it is a low-order point. + `HKDF-SHA256(salt = "c0mpute/job-e2e/v1", ikm = shared)` expands, with the + same length-prefixed binding as `info`, to a spec key and a result key. + +### The sealed spec + +`nonce(12) || AES-256-GCM(spec_key, salt(32) || spec)` with the binding as +associated data, so a sealed spec cannot be replayed into another job, +workload, buyer or worker. The offer's `spec_hash` is +`blake3_derive_key("c0mpute/job-e2e/v1 spec commitment", salt || spec)`; the +winner checks the opened spec against it, and checks that every field of the +offer's `public_spec` equals the same field recomputed from the opened spec, +so a buyer cannot bid out a small job and hand over a big one. It then runs +the workload's bid filter again on the whole spec (all the checks a filter +used to make on a plaintext offer still run before a job starts). Any +failure ends the job with a `Failed` receipt so the buyer re-auctions at +once. + +The buyer holds the sealed spec only until its auction ends, and serves it +only to the peer the accept named; the caller's peer id comes from the noise +handshake. Request-response responses carry up to 10 MiB, so the old +`spec_ref` (large specs stored as a plaintext chunk the winner fetched by +hash) is no longer used by any buyer. + +### Sealed blobs (inputs and results) + +`magic "C0JOBE1\0" || salt(32) || segments`. The blob key is +`HKDF(salt, root)`; segment `i` (64 KiB of plaintext) is AES-256-GCM with +nonce `last(1) || 0(3) || i(8)` and the header as associated data. The last +segment carries the flag, so truncation, reordering and splicing all fail. +Files are sealed and opened streamed. + +- Results: `root` = the result key. `publish_result_bytes` / + `publish_result_file` seal before storing (render frames and text adapters + too). Buyers open through `gateway_util::fetch_result` / + `fetch_result_to_file` and `AuctionOutcome::open_result`. +- Inputs: `root` = a random key per input + (`put_bytes_pieces_sealed` / `put_file_pieces_sealed`), carried as + `_key` (`audio_key`, `image_key`, `input_key`, `corpus_key`, + `adapter_key`) inside the sealed spec. Workers open with + `InputRef::fetch_opened` / `fetch_to_file_opened`. + +Receipts are unchanged: `output_hash` names the ciphertext, which is all a +validator or billing system needs. + +### Public specs + +A per-workload allowlist of small scalar fields, plus size buckets (powers of +two): model, engine, voice, locale, language, width, height, steps, frames, +format, lang, frame range, render engine and device, `max_tokens`, +`base_model`, webhook retry budget, gameserver size, and so on. Never +prompts, texts, messages, URLs, headers, input hashes or keys, file names, +client keys or tokens (`gameserver.stop` shows `sha256(token)`, which is +already the server's public stop hash). Strings over 128 bytes never go in a +public spec whatever the key. An unknown workload shows only `v`. + +## What the winning worker can still see + +The winner must see plaintext to compute on it. It decrypts the spec in +memory, inputs in memory or in the job's scratch directory (removed when the +job ends, on failure paths too), and never logs content. It keeps the +plaintext result only until it is sealed. A sealed `.blend` stays sealed in +the render cache; a sealed adapter is kept only for its job, never in the +adapter cache. + +A dishonest winner can still copy what it computes on. Two stronger options: + +1. **Private mode** (`c0mpute private`, already shipped): bid only for, and + accept only, your own or allowlisted nodes. Use it for sensitive work. +2. **Confidential computing** (`with_confidential(true)` on an auction, + capability `c0mpute:confidential`): only workers running in an attested + TEE (AMD SEV-SNP, Intel TDX) can win. The offer carries a fresh + `attest_nonce`; the bid carries attestation evidence whose report data is + `blake3_xof(job_id, worker_peer, W, nonce)`, binding the TEE to the very + key the spec is sealed to; the buyer verifies it (`AttestationVerifier`) + before sealing. Workers produce evidence through `AttestationProvider` + and only advertise the tag when one is configured. **Shipped:** the + requirement, tags, report-data binding and both hooks, failing closed (a + confidential auction with no verifier gets no winner). **Deferred:** the + concrete SNP/TDX evidence producer and verifier (VCEK / PCS collateral, + measurement policy); no node in the fleet runs in a TEE today. + +## What observers can still see + +Metadata: that a job of a given workload ran, its public spec, its price, +timing, which peers bid and which won, and the sizes (as buckets for the +public spec, exactly for ciphertext chunks). `infernet.text.train` progress +events (step, loss, throughput) still go to gossip under the job's run id. + +## Compatibility + +- New buyers send only e2e offers, with `c0mpute:e2e:v1` in + `required_capabilities`. Workers older than this release check that they + advertise every required tag, so they never bid; buyers ignore bids + without a signed key anyway. +- New workers still serve plaintext offers from old buyers for one release + (`C0MPUTE_REJECT_PLAINTEXT_OFFERS=1` refuses them now). The next release + removes plaintext offers. +- New workers still open inputs without a `_key` (plaintext chunks a + client stored itself, as the jobs bridge allows). + +## Alternatives considered + +- **Encrypt the offer to every eligible worker.** Leaks the job to every + bidder, not just the winner, and needs a worker directory with keys. +- **Seal on the accept and gossip it.** Ciphertext on gossip is safe, but a + multi-MiB spec does not fit a 64 KiB gossip message, and gossip fans it to + every node for nothing. Point to point is smaller and does not need the + `spec_ref` detour. +- **Static worker keys.** Simpler, but a stolen worker key would open every + job it ever won. Per-bid keys signed by the identity give forward secrecy + for the cost of one signature per bid. +- **Transport-only (noise) plus access control on chunk fetch.** Protects + the wire but leaves plaintext at rest on workers and buyers, and relies on + every node enforcing the ACL. diff --git a/dips/README.md b/dips/README.md index 556e657c..2a6232c4 100644 --- a/dips/README.md +++ b/dips/README.md @@ -136,3 +136,4 @@ superseded-by: | 0026 | Infernet temporal policies (liquid networks) as c0mpute workloads | Accepted | | 0030 | `workflow.step` workload and the `c0mpute jobs serve` bridge | Review | | 0031 | Apps: containerised backends with 3x redundancy, git remote | Review | +| 0032 | End-to-end encrypted jobs (sealed spec, inputs and results) | Accepted | diff --git a/docs/protocol/jobs.md b/docs/protocol/jobs.md index e4935c3d..d860103b 100644 --- a/docs/protocol/jobs.md +++ b/docs/protocol/jobs.md @@ -118,6 +118,15 @@ one host disappearing. `encryption` absent means plaintext. Appropriate for public inputs; not for private work. +On the live network every job is end-to-end encrypted +([DIP-0032](../../dips/0032-e2e-encrypted-jobs.md)): gossip offers carry only a +public spec (model, engine, size buckets) and the buyer's fresh X25519 key; the +spec is sealed to the winning worker alone and fetched point to point; inputs +are stored sealed under per-input keys carried inside the sealed spec; results +are sealed back to the buyer. The winning worker still sees the plaintext it +computes on: use private mode or the `confidential` (attested TEE) requirement +for sensitive work. + ## Execution defaults are conservative ```json diff --git a/node/crates/c0mpute-cli/src/main.rs b/node/crates/c0mpute-cli/src/main.rs index 2b4e28b2..4620378b 100644 --- a/node/crates/c0mpute-cli/src/main.rs +++ b/node/crates/c0mpute-cli/src/main.rs @@ -2691,10 +2691,10 @@ async fn run_transcode(cmd: TranscodeCmd) -> Result<()> { if let Some(path) = &output { // Pieced outputs (anything over 1 MiB from a current // worker) are written a piece at a time. + // Sealed to this buyer; opened as it is written. let written = c0mpute_core::gateway_util::fetch_result_to_file( &net, - h, - &outcome.accepted_bid.bidder_peer_id, + &outcome, c0mpute_core::runner::MAX_OUTPUT_BYTES as usize, std::time::Duration::from_secs(30), path, diff --git a/node/crates/c0mpute-core/Cargo.toml b/node/crates/c0mpute-core/Cargo.toml index b6ccc371..2e762e6e 100644 --- a/node/crates/c0mpute-core/Cargo.toml +++ b/node/crates/c0mpute-core/Cargo.toml @@ -41,6 +41,7 @@ x25519-dalek = { workspace = true } hkdf = { workspace = true } hmac = { workspace = true } aes-gcm = { workspace = true } +zeroize = { workspace = true } base64 = { workspace = true } sysinfo = { workspace = true } zip = { workspace = true } diff --git a/node/crates/c0mpute-core/src/buyer.rs b/node/crates/c0mpute-core/src/buyer.rs index 7e78da84..d5962545 100644 --- a/node/crates/c0mpute-core/src/buyer.rs +++ b/node/crates/c0mpute-core/src/buyer.rs @@ -1,10 +1,11 @@ //! Buyer side of the auction: publish a `JobOffer`, collect `JobBid`s, //! pick a winner, publish `JobAccept`, wait for `JobReceipt`. //! -//! Phase 1: inline-spec auction. The full workload spec ships in -//! `JobOffer.spec_inline` (and again in `JobAccept.spec_inline`) so -//! workers never need a separate manifest fetch. Phase 2 swaps this for -//! the chunk-store / request-response transfer keyed off `spec_hash`. +//! Every auction is end-to-end encrypted ([`crate::job_e2e`], DIP-0032): the +//! offer carries only a public spec and a fresh buyer X25519 key, bids carry +//! signed worker keys, the accept carries no spec, and the winner alone +//! fetches the spec sealed to its key over request-response. The outcome +//! holds the key that opens the worker's sealed result. use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -61,6 +62,10 @@ pub struct JobAuction { /// workloads that exchange more with the winner before its receipt /// (`webhook.deliver` seals its request to the winner's key). pub accept_notify: Option>, + /// Require an attested TEE ([`c0mpute_net::topics::CONFIDENTIAL_TAG`]): + /// only bids whose attestation verifies are considered, and the spec is + /// sealed to the attested key only. + pub confidential: bool, } impl JobAuction { @@ -79,9 +84,15 @@ impl JobAuction { external_ref: None, policy: crate::routing::SelectionPolicy::default(), accept_notify: None, + confidential: false, } } + pub fn with_confidential(mut self, confidential: bool) -> Self { + self.confidential = confidential; + self + } + pub fn with_accept_notify( mut self, tx: tokio::sync::mpsc::UnboundedSender<(String, JobBid)>, @@ -169,6 +180,37 @@ pub struct AuctionOutcome { /// The auction's `external_ref`, as the buyer set it. Use this rather /// than `receipt.external_ref`, which workers older than 0.2.49 omit. pub external_ref: Option, + /// Opens the result the worker sealed to us (the chunk the receipt's + /// `output_hash` names is ciphertext). Never leaves this process. + pub result_key: c0mpute_net::topics::JobKey, +} + +impl AuctionOutcome { + /// Open the sealed result bytes fetched by the receipt's hash. + pub fn open_result(&self, sealed: &[u8]) -> Result> { + crate::job_e2e::open_blob(&self.result_key, sealed) + } + + /// Open a sealed result fetched to a file, in place. + pub async fn open_result_file(&self, path: &std::path::Path) -> Result { + crate::job_e2e::open_file_in_place(&self.result_key, path).await + } +} + +/// The offer's required tags: the caller's, plus the e2e tag (so pre-e2e +/// workers never bid) and the TEE tag for confidential work. +fn e2e_required_capabilities(caps: &[String], confidential: bool) -> Vec { + let mut caps = caps.to_vec(); + let mut want = vec![c0mpute_net::topics::JOB_E2E_TAG]; + if confidential { + want.push(c0mpute_net::topics::CONFIDENTIAL_TAG); + } + for t in want { + if !caps.iter().any(|c| c == t) { + caps.push(t.to_string()); + } + } + caps } /// Run the full buyer-side auction synchronously: subscribe → publish @@ -194,14 +236,16 @@ pub async fn run_auction(net: Arc, auction: JobAuction) -> Result let job_id = Uuid::new_v4().to_string(); let now_ms = unix_ms(); + let session = crate::job_e2e::BuyerSession::new(&auction.spec_inline, auction.confidential)?; let offer = JobOffer { job_id: job_id.clone(), workload_type: auction.workload_type.clone(), buyer_peer_id: net.peer_id().to_base58(), buyer_did: None, - spec_hash: blake3_hex(&auction.spec_inline.to_string()), - spec_inline: Some(auction.spec_inline.clone()), - required_capabilities: auction.required_capabilities.clone(), + // A salted commitment: says nothing about the spec to observers. + spec_hash: session.spec_hash(), + spec_inline: None, + required_capabilities: e2e_required_capabilities(&auction.required_capabilities, auction.confidential), max_price_usd: auction.max_price_usd, deadline_unix_ms: now_ms + auction.deadline.as_millis() as u64, published_at_ms: now_ms, @@ -210,6 +254,7 @@ pub async fn run_auction(net: Arc, auction: JobAuction) -> Result // on the gossip mesh, which drops them while it re-forms. buyer_addrs: offer_buyer_addrs(&net).await, bid_window_ms: Some(auction.bid_window.as_millis() as u64), + e2e: Some(session.offer_part(&auction.workload_type, &auction.spec_inline)), }; info!( @@ -290,6 +335,12 @@ pub async fn run_auction(net: Arc, auction: JobAuction) -> Result fallback.push(bid); continue; } + // Only a key the bidder signed for this job (and, for + // confidential work, attested) may receive the spec. + if let Err(e) = session.check_bid(&job_id, &auction.workload_type, &offer.buyer_peer_id, &bid) { + debug!(%job_id, bidder = %bid.bidder_peer_id, err = %e, "buyer: bid without a usable e2e key ignored"); + continue; + } debug!( %job_id, bidder = %bid.bidder_peer_id, @@ -355,6 +406,18 @@ pub async fn run_auction(net: Arc, auction: JobAuction) -> Result "buyer: winner chosen" ); + // ── seal the spec to the winner alone ─────────────────────────── + let winner_peer: c0mpute_net::PeerId = winner + .bidder_peer_id + .parse() + .map_err(|e| anyhow::anyhow!("winner peer id: {e}"))?; + let (sealed, result_key) = + session.seal_for(&job_id, &auction.workload_type, &offer.buyer_peer_id, &winner)?; + drop(session); + // Held until this function returns (receipt or timeout); served only to + // the winner over request-response, never on gossip. + let _payload = crate::job_e2e::hold_payload(&net, &job_id, winner_peer, sealed); + // ── publish accept ────────────────────────────────────────────── let accept = JobAccept { job_id: job_id.clone(), @@ -362,13 +425,15 @@ pub async fn run_auction(net: Arc, auction: JobAuction) -> Result winning_bidder_peer_id: winner.bidder_peer_id.clone(), agreed_price_usd: winner.price_usd, workload_type: auction.workload_type.clone(), - spec_inline: Some(auction.spec_inline.clone()), + spec_inline: None, published_at_ms: unix_ms(), external_ref: auction.external_ref.clone(), buyer_addrs: crate::reachability::buyer_external_addrs( std::env::var("C0MPUTE_P2P_EXTERNAL").ok().as_deref(), &offer.buyer_peer_id, ), + e2e: Some(c0mpute_net::topics::JOB_E2E_VERSION), + result_key: None, }; // Tell the winner directly as well as on the topic: an accept lost on // the mesh used to leave the buyer waiting out the whole receipt timeout. @@ -429,6 +494,7 @@ pub async fn run_auction(net: Arc, auction: JobAuction) -> Result accepted_bid: winner, receipt, external_ref: auction.external_ref.clone(), + result_key, }); } } @@ -498,20 +564,22 @@ pub async fn probe_auction( let mut rx = net.messages(); let job_id = Uuid::new_v4().to_string(); let now_ms = unix_ms(); + let session = crate::job_e2e::BuyerSession::new(&spec_inline, false)?; let offer = JobOffer { job_id: job_id.clone(), workload_type: workload.to_string(), buyer_peer_id: net.peer_id().to_base58(), buyer_did: None, - spec_hash: blake3_hex(&spec_inline.to_string()), - spec_inline: Some(spec_inline), - required_capabilities, + spec_hash: session.spec_hash(), + spec_inline: None, + required_capabilities: e2e_required_capabilities(&required_capabilities, false), max_price_usd: 0.0, deadline_unix_ms: now_ms + 10 * 60 * 1000, published_at_ms: now_ms, external_ref: None, buyer_addrs: offer_buyer_addrs(&net).await, bid_window_ms: Some(window.as_millis() as u64), + e2e: Some(session.offer_part(workload, &spec_inline)), }; publish_json(&net, &topic, &offer).await?; let deadline = Instant::now() + window; @@ -617,11 +685,6 @@ fn unix_ms() -> u64 { .unwrap_or(0) } -fn blake3_hex(s: &str) -> String { - let h = blake3::hash(s.as_bytes()); - hex::encode(h.as_bytes()) -} - // Dummy use to suppress "unused" if `JobStatus` isn't referenced elsewhere // in this module — it's part of the receipt we return. #[allow(dead_code)] @@ -644,6 +707,7 @@ mod tests { expires_at_ms, bench_score: None, load: None, + e2e: None, } } diff --git a/node/crates/c0mpute-core/src/capabilities.rs b/node/crates/c0mpute-core/src/capabilities.rs index fad94012..b96b2a9c 100644 --- a/node/crates/c0mpute-core/src/capabilities.rs +++ b/node/crates/c0mpute-core/src/capabilities.rs @@ -64,7 +64,11 @@ pub fn tags_from_config(config: &Config) -> Vec { /// restart. The hardware tag, which shells out to `nvidia-smi`, is probed /// once and reused. pub fn role_tags(config: &Config) -> Vec { - let mut tags = Vec::new(); + // Not a workload: this node takes end-to-end encrypted jobs (DIP-0032). + let mut tags = vec![crate::job_e2e::JOB_E2E_TAG.to_string()]; + if crate::job_e2e::attestation_provider().is_some() { + tags.push(crate::job_e2e::CONFIDENTIAL_TAG.to_string()); + } // Not a workload: shows on the node's card that it donates idle compute. if crate::crowdgpt::donating() { tags.push(crate::crowdgpt::DONATE_TAG.to_string()); diff --git a/node/crates/c0mpute-core/src/captcha.rs b/node/crates/c0mpute-core/src/captcha.rs index cb7ed7a0..5347eb42 100644 --- a/node/crates/c0mpute-core/src/captcha.rs +++ b/node/crates/c0mpute-core/src/captcha.rs @@ -788,7 +788,7 @@ async fn run_captcha_job( if bytes.len() > MAX_RESULT_BYTES { bail!("captcha result is over the {MAX_RESULT_BYTES}-byte limit"); } - crate::pieces::publish_result_bytes(net, store, &bytes) + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes) .await .context("publish captcha result") } diff --git a/node/crates/c0mpute-core/src/crawl.rs b/node/crates/c0mpute-core/src/crawl.rs index b0a7af57..f245a996 100644 --- a/node/crates/c0mpute-core/src/crawl.rs +++ b/node/crates/c0mpute-core/src/crawl.rs @@ -280,7 +280,7 @@ async fn run_fetch_job(net: &Libp2pNetwork, store: &ChunkStore, job: &AcceptedJo let spec = workflow::resolve_spec(net, job).await?; let result = execute(&spec).await; let bytes = serde_json::to_vec(&result)?; - crate::pieces::publish_result_bytes(net, store, &bytes) + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes) .await .context("publish crawl result") } diff --git a/node/crates/c0mpute-core/src/diffusion.rs b/node/crates/c0mpute-core/src/diffusion.rs index ece8e2db..1c51bf4a 100644 --- a/node/crates/c0mpute-core/src/diffusion.rs +++ b/node/crates/c0mpute-core/src/diffusion.rs @@ -747,7 +747,7 @@ async fn run_job( if !is_png(&bytes) || bytes.len() > MAX_IMAGE_BYTES { bail!("sd-cli output is not a PNG under the 8 MiB result limit"); } - crate::pieces::publish_result_bytes(net, store, &bytes).await + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes).await } #[cfg(test)] diff --git a/node/crates/c0mpute-core/src/dispatch.rs b/node/crates/c0mpute-core/src/dispatch.rs index 97b0cf61..ee3d2889 100644 --- a/node/crates/c0mpute-core/src/dispatch.rs +++ b/node/crates/c0mpute-core/src/dispatch.rs @@ -22,10 +22,13 @@ //! A worker that is draining for an update ([`c0mpute_update::draining`]) //! does not bid at all. //! -//! After winning bid (received as `JobAccept`), the worker fetches the -//! manifest, runs the workload, publishes a `JobReceipt`. The receipt -//! flow lives in a follow-up commit — this module ships the offer/bid -//! plumbing. +//! End-to-end encrypted offers ([`crate::job_e2e`], DIP-0032) carry only a +//! public spec: filters judge that at bid time, and the bid carries a fresh +//! X25519 key signed by this node's identity. On winning, the worker fetches +//! the sealed spec from the buyer point to point, opens it, checks it +//! against the offer's commitment and public spec, runs the workload's filter +//! again on the whole spec, and only then hands the job to the runner (with +//! the key its result is sealed back under). //! //! The buyer side (publishing offers + collecting bids + accepting one) //! lives in `c0mpute job submit` and is not part of the worker daemon. @@ -185,6 +188,11 @@ pub fn run_worker_subscriber_with_load( if offer.workload_type != workload_type { continue; } + // Plaintext offers are served for one release more (DIP-0032). + if offer.e2e.is_none() && !crate::job_e2e::serves_plaintext_offers() { + debug!(job_id = %offer.job_id, "dispatch: plaintext offer refused (e2e only)"); + continue; + } // Private mode: only trusted buyers (by signed publisher). if !crate::private::current().admits(&msg, &offer, &our_peer_id) { debug!(job_id = %offer.job_id, buyer = %offer.buyer_peer_id, "dispatch: private mode, buyer not trusted"); @@ -234,7 +242,7 @@ pub fn run_worker_subscriber_with_load( capacity.total_slots, load.as_ref().and_then(WorkerLoad::recent_job_duration), ); - if let Some((price, expires_at_ms, payload)) = handle_offer( + if let Some((price, expires_at_ms, payload, session)) = handle_offer( &net, &topic, &offer, @@ -264,6 +272,7 @@ pub fn run_worker_subscriber_with_load( expires_at_ms, reserve_until_ms: reserve_until(&offer, now, expires_at_ms), payload: Some(payload), + e2e: session.map(|s| (Arc::new(s), Arc::new(offer.clone()))), }, ); } @@ -321,6 +330,12 @@ pub fn run_worker_subscriber_with_load( "dispatch: we won — forwarding to runner" ); crate::activity::job_started(&accept.job_id); + if let Some((session, offer)) = bid.e2e { + // Fetching the sealed spec takes a round trip to the + // buyer: never in the loop that reads offers. + spawn_open_e2e(net.clone(), tx.clone(), filter.clone(), session, offer, accept); + continue; + } if let Err(e) = tx.send(AcceptedJob { accept }).await { warn!(err = %e, "dispatch: runner channel closed"); } @@ -344,6 +359,8 @@ pub(crate) struct OpenBid { pub reserve_until_ms: u64, /// The serialized bid, re-sent if the buyer re-sends its offer. pub payload: Option>, + /// For a bid on an e2e offer: our X25519 secret for it and the offer. + pub e2e: Option<(Arc, Arc)>, } impl OpenBid { @@ -364,7 +381,12 @@ impl OpenBid { if accept.buyer_peer_id != self.buyer_peer_id { return Err("buyer differs from the offer"); } - if accept.spec_inline != self.spec_inline { + if self.e2e.is_some() { + // The spec comes sealed after the accept, never on it. + if accept.e2e != Some(c0mpute_net::topics::JOB_E2E_VERSION) || accept.spec_inline.is_some() { + return Err("e2e offer answered by a plaintext accept"); + } + } else if accept.spec_inline != self.spec_inline { return Err("spec differs from the offer we bid on"); } if accept.external_ref != self.external_ref { @@ -553,7 +575,7 @@ async fn handle_offer( eta_seconds: u64, bench_score: Option, bid_load: f32, -) -> Option<(f64, u64, Vec)> { +) -> Option<(f64, u64, Vec, Option)> { let now_ms = unix_ms(); if now_ms.saturating_sub(offer.published_at_ms) > MAX_OFFER_AGE.as_millis() as u64 { debug!(job_id = %offer.job_id, "dispatch: ignoring stale offer"); @@ -575,7 +597,7 @@ async fn handle_offer( let missing: Vec<&String> = offer .required_capabilities .iter() - .filter(|c| !advertised_tags.iter().any(|a| a == *c)) + .filter(|c| !crate::job_e2e::implicit_tag(c) && !advertised_tags.iter().any(|a| a == *c)) .collect(); if !missing.is_empty() { debug!( @@ -598,6 +620,17 @@ async fn handle_offer( }, None => offer.max_price_usd, }; + // A fresh X25519 key for this one job, signed by our identity. + let (session, e2e) = match offer.e2e.as_ref() { + Some(_) => match crate::job_e2e::WorkerSession::for_offer(net, offer) { + Ok((s, e)) => (Some(s), Some(e)), + Err(e) => { + debug!(job_id = %offer.job_id, err = %e, "dispatch: cannot bid on this e2e offer"); + return None; + } + }, + None => (None, None), + }; let expires_at_ms = now_ms + BID_TTL.as_millis() as u64; let bid = JobBid { job_id: offer.job_id.clone(), @@ -609,6 +642,7 @@ async fn handle_offer( expires_at_ms: Some(expires_at_ms), bench_score, load: Some(bid_load), + e2e, }; let payload = match serde_json::to_vec(&bid) { Ok(b) => b, @@ -624,7 +658,43 @@ async fn handle_offer( "dispatch: bid placed" ); spawn_deliver_bid(net.clone(), topic.to_string(), offer, payload.clone()); - Some((price_usd, expires_at_ms, payload)) + Some((price_usd, expires_at_ms, payload, session)) +} + +/// Won an e2e job: fetch and open the sealed spec, check it, re-run the +/// workload's filter on the whole spec, then hand the job to the runner with +/// its result key. Any failure ends the job with a Failed receipt, so the +/// buyer re-auctions at once instead of waiting out its timeout. Nothing +/// about the spec's content is logged. +fn spawn_open_e2e( + net: Arc, + tx: mpsc::Sender, + filter: Option, + session: Arc, + offer: Arc, + mut accept: JobAccept, +) { + tokio::spawn(async move { + let opened = session.open_accept(&net, &offer, &accept).await; + drop(session); + let refused = match opened { + Ok((spec, key)) => { + let full = crate::job_e2e::opened_offer(&offer, spec); + if filter.as_ref().is_none_or(|f| f(&full)) { + accept.spec_inline = full.spec_inline; + accept.result_key = Some(key); + if let Err(e) = tx.send(AcceptedJob { accept }).await { + warn!(err = %e, "dispatch: runner channel closed"); + } + return; + } + "the sealed spec does not pass this workload's checks".to_string() + } + Err(e) => format!("{e:#}"), + }; + warn!(job_id = %accept.job_id, reason = %refused, "dispatch: won e2e job refused"); + crate::runner::publish_receipt(&net, &accept, None, c0mpute_net::topics::JobStatus::Failed).await; + }); } /// Workload types this worker should listen for, derived from its @@ -694,6 +764,7 @@ mod tests { expires_at_ms: NOW + 60_000, reserve_until_ms: NOW + 18_000, payload: None, + e2e: None, } } @@ -710,6 +781,8 @@ mod tests { published_at_ms: 0, external_ref: Some("infernet:job_42".into()), buyer_addrs: Vec::new(), + e2e: None, + result_key: None, } } @@ -732,6 +805,30 @@ mod tests { assert!(bid().matches(&a, NOW).is_err()); } + #[test] + fn e2e_bids_take_only_e2e_accepts_without_a_spec() { + let mut b = bid(); + b.spec_inline = None; + let offer = offer_at(NOW, None); + // A session is only needed as a marker here. + let session = std::sync::Arc::new(test_session()); + b.e2e = Some((session, std::sync::Arc::new(offer))); + let mut a = accept(); + // A plaintext accept for an e2e bid (a downgrade) is refused. + assert!(b.matches(&a, NOW).is_err()); + a.spec_inline = None; + assert!(b.matches(&a, NOW).is_err(), "missing e2e version"); + a.e2e = Some(c0mpute_net::topics::JOB_E2E_VERSION); + assert!(b.matches(&a, NOW).is_ok()); + // Any spec on an e2e accept is refused, even with the version set. + a.spec_inline = Some(serde_json::json!({"x": 1})); + assert!(b.matches(&a, NOW).is_err()); + } + + fn test_session() -> crate::job_e2e::WorkerSession { + crate::job_e2e::WorkerSession::for_tests() + } + #[test] fn accept_over_cap_or_nan_is_rejected() { let mut a = accept(); @@ -790,6 +887,7 @@ mod tests { external_ref: None, buyer_addrs: Vec::new(), bid_window_ms: window, + e2e: None, } } diff --git a/node/crates/c0mpute-core/src/embed.rs b/node/crates/c0mpute-core/src/embed.rs index 8b3ae60a..f87087b4 100644 --- a/node/crates/c0mpute-core/src/embed.rs +++ b/node/crates/c0mpute-core/src/embed.rs @@ -498,7 +498,7 @@ async fn run_embed_job( if bytes.len() > MAX_RESULT_BYTES { bail!("embeddings result is over the {MAX_RESULT_BYTES}-byte limit"); } - crate::pieces::publish_result_bytes(net, store, &bytes) + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes) .await .context("publish embeddings result") } diff --git a/node/crates/c0mpute-core/src/gameserver.rs b/node/crates/c0mpute-core/src/gameserver.rs index ed9dca53..ba85370f 100644 --- a/node/crates/c0mpute-core/src/gameserver.rs +++ b/node/crates/c0mpute-core/src/gameserver.rs @@ -1138,7 +1138,21 @@ impl Worker { } pub fn accepts_start(&self, offer: &JobOffer) -> bool { - let Some(spec) = offer.spec_inline.as_ref().and_then(|v| serde_json::from_value::(v.clone()).ok()) else { + // An e2e offer keeps its keys, stop hash and world sealed: stand-ins + // let the size and price checks run; the opened spec is checked again. + let spec = match crate::job_e2e::bid_spec(offer) { + Some(crate::job_e2e::BidSpec::Public(v)) => Some(crate::job_e2e::with_placeholders( + v, + &[ + ("stop_hash", serde_json::json!(format!("sha256:{}", "0".repeat(64)))), + ("client_pubkey", serde_json::json!("1".repeat(64))), + ("backup_pubkey", serde_json::json!("1".repeat(64))), + ], + )), + Some(crate::job_e2e::BidSpec::Full(v)) => Some(v.clone()), + None => None, + }; + let Some(spec) = spec.and_then(|v| serde_json::from_value::(v).ok()) else { return false; }; if spec.validate().is_err() { @@ -1157,6 +1171,13 @@ impl Worker { } pub fn accepts_stop(&self, offer: &JobOffer) -> bool { + // An e2e offer shows the token's hash, never the token. + if let Some(crate::job_e2e::BidSpec::Public(v)) = crate::job_e2e::bid_spec(offer) { + let (Some(id), Some(h)) = (v["id"].as_str(), v["token_hash"].as_str()) else { + return false; + }; + return self.snapshot().get(id).is_some_and(|s| s.stop_hash == h); + } let Some(spec) = offer.spec_inline.as_ref().and_then(|v| serde_json::from_value::(v.clone()).ok()) else { return false; }; @@ -1418,10 +1439,9 @@ impl Worker { } } - async fn publish_result(&self, v: &Value) -> Result { - let h = self.store.put(&serde_json::to_vec(v)?).await?; - self.net.announce(&h).await?; - Ok(h.to_hex()) + /// Store the job's result, sealed to the buyer for e2e jobs (DIP-0032). + async fn publish_result(&self, accept: &c0mpute_net::topics::JobAccept, v: &Value) -> Result { + crate::pieces::publish_result_bytes(&self.net, &self.store, accept, &serde_json::to_vec(v)?).await } } @@ -1435,7 +1455,7 @@ fn spawn_runner(worker: Arc, slots: Arc, stop: bool) -> mpsc: let _slot = slot; let res = if stop { w.stop(&job).await } else { w.start(&job).await }; let res = match res { - Ok(v) => w.publish_result(&v).await, + Ok(v) => w.publish_result(&job.accept, &v).await, Err(e) => Err(e), }; match res { @@ -1895,8 +1915,8 @@ async fn auction( if o.receipt.status != JobStatus::Completed { bail!("worker {} reported {:?}", o.accepted_bid.bidder_peer_id, o.receipt.status); } - let hash = o.receipt.output_hash.clone().context("worker returned no result hash")?; - let bytes = crate::gateway_util::fetch_result(&net, &hash, &o.accepted_bid.bidder_peer_id, 64 * 1024, Duration::from_secs(30)).await.map_err(|e| anyhow::anyhow!("fetch result: {e:?}"))?; + o.receipt.output_hash.as_ref().context("worker returned no result hash")?; + let bytes = crate::gateway_util::fetch_result(&net, &o, 64 * 1024, Duration::from_secs(30)).await.map_err(|e| anyhow::anyhow!("fetch result: {e:?}"))?; let v: Value = serde_json::from_slice(&bytes)?; if v["ok"] != json!(true) { bail!("worker result: {v}"); diff --git a/node/crates/c0mpute-core/src/gateway_util.rs b/node/crates/c0mpute-core/src/gateway_util.rs index 6d27087e..153da48b 100644 --- a/node/crates/c0mpute-core/src/gateway_util.rs +++ b/node/crates/c0mpute-core/src/gateway_util.rs @@ -162,13 +162,63 @@ fn piece_error(error: anyhow::Error) -> FetchError { } } -/// Fetch a worker's result by the hash its receipt named, verified, within -/// `max_bytes` and `budget`. Workers return results up to +/// Fetch and open the result of an auction: the chunk its receipt names is +/// sealed to us ([`crate::job_e2e`]); it is fetched within the sealed size +/// of `max_bytes`, opened with the auction's result key, and the plaintext +/// checked against `max_bytes`. +pub async fn fetch_result( + net: &std::sync::Arc, + outcome: &crate::AuctionOutcome, + max_bytes: usize, + budget: Duration, +) -> Result, FetchError> { + let hash = outcome + .receipt + .output_hash + .as_deref() + .ok_or(FetchError::Invalid("worker returned no output hash"))?; + let sealed_max = crate::job_e2e::sealed_len(max_bytes as u64) as usize; + let sealed = fetch_stored_result(net, hash, &outcome.accepted_bid.bidder_peer_id, sealed_max, budget).await?; + let plain = outcome + .open_result(&sealed) + .map_err(|_| FetchError::Invalid("worker result does not open with this job's key"))?; + if plain.len() > max_bytes { + return Err(FetchError::Invalid("worker result is over the size cap")); + } + Ok(plain) +} + +/// [`fetch_result`] straight to `dest`: the sealed result is fetched to a +/// file next to it (pieced results a piece at a time) and opened into `dest` +/// streamed. Returns the plaintext size. +pub async fn fetch_result_to_file( + net: &std::sync::Arc, + outcome: &crate::AuctionOutcome, + max_bytes: usize, + root_budget: Duration, + dest: &std::path::Path, +) -> anyhow::Result { + let hash = outcome.receipt.output_hash.as_deref().ok_or_else(|| anyhow::anyhow!("worker returned no output hash"))?; + let sealed = dest.with_extension("sealed"); + let sealed_max = crate::job_e2e::sealed_len(max_bytes as u64) as usize; + let fetched = + fetch_stored_result_to_file(net, hash, &outcome.accepted_bid.bidder_peer_id, sealed_max, root_budget, &sealed).await; + let opened = match fetched { + Ok(_) => crate::job_e2e::open_file(&outcome.result_key, &sealed, dest).await, + Err(e) => Err(e), + }; + let _ = tokio::fs::remove_file(&sealed).await; + opened +} + +/// Fetch the chunk a receipt named, verified, within `max_bytes` and +/// `budget`. Workers return results up to /// [`crate::pieces::RESULT_SINGLE_CHUNK_MAX`] as that one chunk (so do all /// workers that predate pieces, at any size); bigger results from current /// workers are a manifest whose pieces are fetched a few at a time and each /// checked, then the whole hash and size. -pub async fn fetch_result( +/// Fetch the chunk (or manifest and pieces) a receipt names, as stored. +pub async fn fetch_stored_result( net: &std::sync::Arc, hash: &str, provider_peer_id: &str, @@ -188,14 +238,14 @@ pub async fn fetch_result( } } -/// [`fetch_result`] straight to `dest` (a piece at a time for a pieced +/// [`fetch_stored_result`] straight to `dest` (a piece at a time for a pieced /// result, so a multi-GiB output never sits in memory; written under a temp /// name and renamed into place once every check passed). `root_budget` /// bounds the receipt's chunk; each piece then has its own deadline /// ([`crate::pieces::PIECE_FETCH_TIMEOUT`]) rather than one overall cap, so a /// big download is not cut off while it is still making progress. Returns /// the size written. -pub async fn fetch_result_to_file( +pub async fn fetch_stored_result_to_file( net: &std::sync::Arc, hash: &str, provider_peer_id: &str, diff --git a/node/crates/c0mpute-core/src/inference.rs b/node/crates/c0mpute-core/src/inference.rs index dea76cab..8d10f9ee 100644 --- a/node/crates/c0mpute-core/src/inference.rs +++ b/node/crates/c0mpute-core/src/inference.rs @@ -24,7 +24,7 @@ use std::time::{Duration, Instant}; use anyhow::{Context, Result}; use c0mpute_net::topics::JobStatus; -use c0mpute_net::{Libp2pNetwork, Network}; +use c0mpute_net::Libp2pNetwork; use c0mpute_store::ChunkStore; use serde::Deserialize; use serde_json::{Value, json}; @@ -241,8 +241,16 @@ pub fn worker_load() -> WorkerLoad { pub fn offer_filter() -> crate::dispatch::OfferFilter { Arc::new(|offer| { let ready = ready_models(); - !ready.is_empty() - && offer.spec_inline.as_ref().and_then(|v| parse_spec(v).ok()).is_some_and(|s| s.model.is_none_or(|m| ready.contains(&m))) + if ready.is_empty() { + return false; + } + match crate::job_e2e::bid_spec(offer) { + // An e2e offer shows only the model; the whole spec is checked + // again here once it is opened. + Some(crate::job_e2e::BidSpec::Public(v)) => v.get("model").and_then(Value::as_str).is_none_or(|m| ready.iter().any(|r| r == m)), + Some(crate::job_e2e::BidSpec::Full(v)) => parse_spec(v).ok().is_some_and(|s| s.model.is_none_or(|m| ready.contains(&m))), + None => false, + } }) } @@ -260,9 +268,8 @@ pub fn spawn_inference_runner(net: Arc, store: ChunkStore, base: let spec = crate::workflow::resolve_spec(&net, &job).await?; let answer = execute(&base, &spec, &ready_models()).await; let bytes = serde_json::to_vec(&answer)?; - let hash = store.put(&bytes).await.context("store answer")?; - net.announce(&hash).await.context("announce answer")?; - Ok(hash.to_hex()) + // Sealed to the buyer (e2e jobs) before it is stored. + crate::pieces::publish_result_bytes(&net, &store, &job.accept, &bytes).await.context("store answer") } .await; match result { diff --git a/node/crates/c0mpute-core/src/job_e2e.rs b/node/crates/c0mpute-core/src/job_e2e.rs new file mode 100644 index 00000000..3edf3031 --- /dev/null +++ b/node/crates/c0mpute-core/src/job_e2e.rs @@ -0,0 +1,1457 @@ +//! End-to-end encrypted jobs (DIP-0032). +//! +//! Job offers go to gossip topics that every subscriber and relay can read, +//! so an offer carries only what bidding needs. The spec, the inputs and the +//! result travel encrypted between the buyer and the one worker that won: +//! +//! ```text +//! buyer -> gossip offer public_spec (model, engine, size buckets), buyer X25519 key B, +//! spec_hash = salted commitment to the spec +//! worker -> buyer bid worker X25519 key W, signed by the worker's libp2p identity +//! buyer -> both accept names the winner, carries NO spec +//! winner -> buyer rpc `c0mpute.job.payload.v1`: the buyer answers only the winner, +//! with the spec sealed to W (point to point, never gossip) +//! winner -> chunk result sealed to the buyer; the receipt names the ciphertext hash +//! ``` +//! +//! Keys: X25519(B, W) -> HKDF-SHA256 (salt = domain, info = label plus a +//! length-prefixed binding of job id, workload, both peer ids and both keys) +//! -> one AES-256-GCM key for the spec and one for the result. The sealed spec +//! has the same binding as associated data, so it cannot be moved to another +//! job, workload, buyer or worker. Both X25519 keys are fresh per auction and +//! per bid, so a later key compromise opens nothing old. +//! +//! Inputs too large for the spec (audio, images, documents, corpora, .blend +//! files) are sealed by the buyer under a random per-input key before they +//! are stored as chunks; the key rides inside the sealed spec. Results are +//! sealed by the worker under the result key before they are stored. Chunks +//! are therefore ciphertext on every node and on the wire, whoever fetches +//! them. +//! +//! What this does not hide: the winning worker sees the plaintext it computes +//! on (it is decrypted in memory and in the job's scratch directory, which is +//! removed when the job ends). Sensitive work uses private mode (own or +//! allowlisted nodes) or the `confidential` requirement (an attested TEE, see +//! [`AttestationVerifier`]). + +use std::collections::HashMap; +use std::io::{Read, Write}; +use std::path::Path; +use std::sync::{Arc, Mutex, OnceLock, RwLock}; +use std::time::{Duration, Instant}; + +use aes_gcm::aead::{Aead, KeyInit, Payload}; +use aes_gcm::{Aes256Gcm, Nonce}; +use anyhow::{Context, Result, anyhow, bail}; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as B64; +use c0mpute_net::topics::{BidE2e, JOB_E2E_VERSION, JobAccept, JobBid, JobKey, JobOffer, OfferE2e}; +use c0mpute_net::{Libp2pNetwork, PeerId, RpcHandler}; +use hkdf::Hkdf; +use rand::RngCore; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use x25519_dalek::{PublicKey, StaticSecret}; +use zeroize::Zeroizing; + +pub use c0mpute_net::topics::{CONFIDENTIAL_TAG, JOB_E2E_TAG}; + +/// Domain separator for every key, nonce context and signature here. +pub const DOMAIN: &[u8] = b"c0mpute/job-e2e/v1"; +/// Request-response service the winner fetches its sealed spec from. +pub const PAYLOAD_SERVICE: &str = "c0mpute.job.payload.v1"; +/// Largest sealed spec a worker accepts (the largest spec is 1 MiB). +pub const MAX_SEALED_SPEC: usize = 4 * 1024 * 1024; +/// How long a buyer keeps a sealed spec for its winner to fetch. +const PAYLOAD_TTL: Duration = Duration::from_secs(10 * 60); +/// Plaintext bytes per sealed segment of a blob. +pub const BLOB_SEGMENT: usize = 64 * 1024; +const TAG_LEN: usize = 16; +const BLOB_MAGIC: &[u8; 8] = b"C0JOBE1\0"; +/// Bytes before the first segment of a sealed blob: magic and salt. +pub const BLOB_HEADER: usize = 8 + 32; +/// Strings longer than this never go in a public spec, whatever the key. +const MAX_PUBLIC_STRING: usize = 128; + +// ─── bindings and keys ─────────────────────────────────────────────────── + +/// Everything a job's keys and ciphertexts are bound to. +#[derive(Clone, Copy, Debug)] +pub struct Binding<'a> { + pub job_id: &'a str, + pub workload: &'a str, + pub buyer_peer: &'a str, + pub worker_peer: &'a str, + /// Buyer X25519 public key, hex. + pub buyer_key: &'a str, + /// Worker X25519 public key, hex. + pub worker_key: &'a str, +} + +fn lp(out: &mut Vec, b: &[u8]) { + out.extend_from_slice(&(b.len() as u32).to_be_bytes()); + out.extend_from_slice(b); +} + +impl Binding<'_> { + /// Unambiguous encoding: every field length-prefixed, under `label`. + pub fn encode(&self, label: &[u8]) -> Vec { + let mut out = Vec::with_capacity(256); + lp(&mut out, DOMAIN); + lp(&mut out, label); + for f in [ + self.job_id, + self.workload, + self.buyer_peer, + self.worker_peer, + self.buyer_key, + self.worker_key, + ] { + lp(&mut out, f.as_bytes()); + } + out + } +} + +/// The statement a worker signs with its libp2p identity to bind its X25519 +/// key to one job (worker fields of `b` are the worker's own). +pub fn bid_statement(b: &Binding) -> Vec { + b.encode(b"bid") +} + +/// The two keys of one job. +pub struct JobKeys { + pub spec: JobKey, + pub result: JobKey, +} + +pub fn parse_public(hex_key: &str) -> Result { + let b: [u8; 32] = hex::decode(hex_key) + .ok() + .and_then(|v| v.try_into().ok()) + .ok_or_else(|| anyhow!("X25519 key must be 32 bytes of hex"))?; + Ok(PublicKey::from(b)) +} + +pub fn public_hex(secret: &StaticSecret) -> String { + hex::encode(PublicKey::from(secret).as_bytes()) +} + +/// Derive the spec and result keys from our secret and the peer's public key. +pub fn derive(secret: &StaticSecret, peer_key_hex: &str, b: &Binding) -> Result { + let shared = secret.diffie_hellman(&parse_public(peer_key_hex)?); + if !shared.was_contributory() { + bail!("peer X25519 key is a low-order point"); + } + let hk = Hkdf::::new(Some(DOMAIN), shared.as_bytes()); + let mut spec = [0u8; 32]; + let mut result = [0u8; 32]; + hk.expand(&b.encode(b"spec-key"), &mut spec) + .map_err(|_| anyhow!("hkdf"))?; + hk.expand(&b.encode(b"result-key"), &mut result) + .map_err(|_| anyhow!("hkdf"))?; + Ok(JobKeys { + spec: JobKey(spec), + result: JobKey(result), + }) +} + +pub fn random_key() -> JobKey { + let mut k = [0u8; 32]; + rand::rngs::OsRng.fill_bytes(&mut k); + JobKey(k) +} + +pub fn key_from_hex(h: &str) -> Result { + let b: [u8; 32] = hex::decode(h) + .ok() + .and_then(|v| v.try_into().ok()) + .ok_or_else(|| anyhow!("input key must be 32 bytes of hex"))?; + Ok(JobKey(b)) +} + +pub fn key_to_hex(k: &JobKey) -> String { + hex::encode(k.0) +} + +// ─── the sealed spec ───────────────────────────────────────────────────── + +/// The buyer's commitment to a spec, published as the offer's `spec_hash`: +/// keyed by a random salt, so the hash says nothing about a guessable spec. +pub struct Commitment { + salt: [u8; 32], + spec: Zeroizing>, +} + +fn commit_hash(salt: &[u8], spec: &[u8]) -> String { + let mut h = blake3::Hasher::new_derive_key("c0mpute/job-e2e/v1 spec commitment"); + h.update(salt); + h.update(spec); + h.finalize().to_hex().to_string() +} + +impl Commitment { + pub fn new(spec: &Value) -> Result { + let mut salt = [0u8; 32]; + rand::rngs::OsRng.fill_bytes(&mut salt); + Ok(Self { + salt, + spec: Zeroizing::new(serde_json::to_vec(spec)?), + }) + } + + pub fn hash_hex(&self) -> String { + commit_hash(&self.salt, &self.spec) + } +} + +fn aead(key: &JobKey) -> Aes256Gcm { + Aes256Gcm::new_from_slice(&key.0).expect("32-byte key") +} + +/// Seal the committed spec to the winner: `nonce || AES-256-GCM(salt || spec)` +/// with the binding as associated data. +pub fn seal_spec(keys: &JobKeys, b: &Binding, c: &Commitment) -> Result> { + let mut plain = Zeroizing::new(Vec::with_capacity(32 + c.spec.len())); + plain.extend_from_slice(&c.salt); + plain.extend_from_slice(&c.spec); + let mut nonce = [0u8; 12]; + rand::rngs::OsRng.fill_bytes(&mut nonce); + let aad = b.encode(b"spec"); + let ct = aead(&keys.spec) + .encrypt( + Nonce::from_slice(&nonce), + Payload { + msg: &plain, + aad: &aad, + }, + ) + .map_err(|_| anyhow!("seal spec"))?; + let mut out = nonce.to_vec(); + out.extend(ct); + Ok(out) +} + +/// Open a sealed spec and check it against the offer's commitment. Returns +/// the spec bytes. +pub fn open_spec( + keys: &JobKeys, + b: &Binding, + sealed: &[u8], + commitment: &str, +) -> Result>> { + if sealed.len() < 12 + TAG_LEN + 32 || sealed.len() > MAX_SEALED_SPEC { + bail!("sealed spec has an impossible size"); + } + let aad = b.encode(b"spec"); + let plain = Zeroizing::new( + aead(&keys.spec) + .decrypt( + Nonce::from_slice(&sealed[..12]), + Payload { + msg: &sealed[12..], + aad: &aad, + }, + ) + .map_err(|_| anyhow!("sealed spec does not open (wrong key, wrong job or tampered)"))?, + ); + let (salt, spec) = plain.split_at(32); + if commit_hash(salt, spec) != commitment { + bail!("sealed spec does not match the offer's commitment"); + } + Ok(Zeroizing::new(spec.to_vec())) +} + +// ─── sealed blobs (inputs and results) ─────────────────────────────────── +// +// `magic(8) || salt(32) || segment*`. The blob key is HKDF(root, salt), so +// every blob has its own key even under one root; segment i of up to +// BLOB_SEGMENT plaintext bytes is AES-256-GCM with nonce +// `last(1) || 0(3) || i(8, BE)` and the header as associated data. The +// final segment carries the last flag, so truncation and reordering fail. + +/// Size of `plain` bytes once sealed. +pub fn sealed_len(plain: u64) -> u64 { + let segments = plain.div_ceil(BLOB_SEGMENT as u64).max(1); + BLOB_HEADER as u64 + plain + segments * TAG_LEN as u64 +} + +fn blob_cipher(root: &JobKey, salt: &[u8]) -> Aes256Gcm { + let hk = Hkdf::::new(Some(salt), &root.0); + let mut k = Zeroizing::new([0u8; 32]); + hk.expand(b"c0mpute/job-e2e/v1 blob", &mut *k) + .expect("32-byte okm"); + Aes256Gcm::new_from_slice(&*k).expect("32-byte key") +} + +fn seg_nonce(i: u64, last: bool) -> [u8; 12] { + let mut n = [0u8; 12]; + n[0] = u8::from(last); + n[4..].copy_from_slice(&i.to_be_bytes()); + n +} + +fn new_header() -> [u8; BLOB_HEADER] { + let mut h = [0u8; BLOB_HEADER]; + h[..8].copy_from_slice(BLOB_MAGIC); + rand::rngs::OsRng.fill_bytes(&mut h[8..]); + h +} + +/// Seal bytes held in memory. +pub fn seal_blob(root: &JobKey, plain: &[u8]) -> Result> { + let header = new_header(); + let cipher = blob_cipher(root, &header[8..]); + let mut out = Vec::with_capacity(sealed_len(plain.len() as u64) as usize); + out.extend_from_slice(&header); + let segments = plain.len().div_ceil(BLOB_SEGMENT).max(1); + for i in 0..segments { + let seg = + &plain[(i * BLOB_SEGMENT).min(plain.len())..((i + 1) * BLOB_SEGMENT).min(plain.len())]; + let last = i + 1 == segments; + let ct = cipher + .encrypt( + Nonce::from_slice(&seg_nonce(i as u64, last)), + Payload { + msg: seg, + aad: &header, + }, + ) + .map_err(|_| anyhow!("seal blob"))?; + out.extend(ct); + } + Ok(out) +} + +/// Open a sealed blob held in memory. +pub fn open_blob(root: &JobKey, sealed: &[u8]) -> Result> { + if sealed.len() < BLOB_HEADER + TAG_LEN || &sealed[..8] != BLOB_MAGIC { + bail!("not a sealed job blob"); + } + let (header, body) = sealed.split_at(BLOB_HEADER); + let cipher = blob_cipher(root, &header[8..]); + let full = BLOB_SEGMENT + TAG_LEN; + let segments = body.len().div_ceil(full).max(1); + let mut out = Vec::with_capacity(body.len()); + for i in 0..segments { + let seg = &body[i * full..((i + 1) * full).min(body.len())]; + if seg.len() < TAG_LEN { + bail!("sealed blob is truncated"); + } + let last = i + 1 == segments; + let pt = cipher + .decrypt( + Nonce::from_slice(&seg_nonce(i as u64, last)), + Payload { + msg: seg, + aad: header, + }, + ) + .map_err(|_| anyhow!("sealed blob does not open (wrong key, tampered or truncated)"))?; + out.extend(pt); + } + Ok(out) +} + +/// Read until `buf` is full or the reader ends; returns bytes read. +fn read_full(r: &mut impl Read, buf: &mut [u8]) -> std::io::Result { + let mut n = 0; + while n < buf.len() { + match r.read(&mut buf[n..])? { + 0 => break, + k => n += k, + } + } + Ok(n) +} + +fn seal_file_sync(root: &JobKey, src: &Path, dst: &Path) -> Result { + let mut input = std::io::BufReader::new( + std::fs::File::open(src).with_context(|| format!("open {}", src.display()))?, + ); + let mut output = std::io::BufWriter::new( + std::fs::File::create(dst).with_context(|| format!("create {}", dst.display()))?, + ); + let header = new_header(); + let cipher = blob_cipher(root, &header[8..]); + output.write_all(&header)?; + let mut written = BLOB_HEADER as u64; + let mut cur = Zeroizing::new(vec![0u8; BLOB_SEGMENT]); + let mut next = Zeroizing::new(vec![0u8; BLOB_SEGMENT]); + let mut cur_len = read_full(&mut input, &mut cur)?; + let mut i = 0u64; + loop { + // Read ahead: a segment is the last one when nothing follows it. + let next_len = if cur_len == BLOB_SEGMENT { + read_full(&mut input, &mut next)? + } else { + 0 + }; + let last = next_len == 0; + let ct = cipher + .encrypt( + Nonce::from_slice(&seg_nonce(i, last)), + Payload { + msg: &cur[..cur_len], + aad: &header, + }, + ) + .map_err(|_| anyhow!("seal blob"))?; + output.write_all(&ct)?; + written += ct.len() as u64; + if last { + break; + } + std::mem::swap(&mut cur, &mut next); + cur_len = next_len; + i += 1; + } + output.flush()?; + Ok(written) +} + +fn open_file_sync(root: &JobKey, src: &Path, dst: &Path) -> Result { + let mut input = std::io::BufReader::new( + std::fs::File::open(src).with_context(|| format!("open {}", src.display()))?, + ); + let mut header = [0u8; BLOB_HEADER]; + if read_full(&mut input, &mut header)? != BLOB_HEADER || &header[..8] != BLOB_MAGIC { + bail!("not a sealed job blob"); + } + let cipher = blob_cipher(root, &header[8..]); + let mut output = std::io::BufWriter::new( + std::fs::File::create(dst).with_context(|| format!("create {}", dst.display()))?, + ); + let full = BLOB_SEGMENT + TAG_LEN; + let mut cur = vec![0u8; full]; + let mut next = vec![0u8; full]; + let mut cur_len = read_full(&mut input, &mut cur)?; + let mut i = 0u64; + let mut written = 0u64; + loop { + let next_len = if cur_len == full { + read_full(&mut input, &mut next)? + } else { + 0 + }; + let last = next_len == 0; + if cur_len < TAG_LEN { + bail!("sealed blob is truncated"); + } + let pt = Zeroizing::new( + cipher + .decrypt( + Nonce::from_slice(&seg_nonce(i, last)), + Payload { + msg: &cur[..cur_len], + aad: &header, + }, + ) + .map_err(|_| { + anyhow!("sealed blob does not open (wrong key, tampered or truncated)") + })?, + ); + output.write_all(&pt)?; + written += pt.len() as u64; + if last { + break; + } + std::mem::swap(&mut cur, &mut next); + cur_len = next_len; + i += 1; + } + output.flush()?; + Ok(written) +} + +/// Seal the file at `src` into `dst`, streamed. Returns the sealed size. +pub async fn seal_file(root: &JobKey, src: &Path, dst: &Path) -> Result { + let (root, src, dst) = (root.clone(), src.to_path_buf(), dst.to_path_buf()); + tokio::task::spawn_blocking(move || { + let r = seal_file_sync(&root, &src, &dst); + if r.is_err() { + let _ = std::fs::remove_file(&dst); + } + r + }) + .await? +} + +/// Open the sealed file at `src` into `dst`, streamed. `dst` is removed when +/// any segment fails to open. Returns the plaintext size. +pub async fn open_file(root: &JobKey, src: &Path, dst: &Path) -> Result { + let (root, src, dst) = (root.clone(), src.to_path_buf(), dst.to_path_buf()); + tokio::task::spawn_blocking(move || { + let r = open_file_sync(&root, &src, &dst); + if r.is_err() { + let _ = std::fs::remove_file(&dst); + } + r + }) + .await? +} + +/// Open a blob in place on disk (`path` holds the sealed bytes on entry and +/// the plaintext on success). +pub async fn open_file_in_place(root: &JobKey, path: &Path) -> Result { + let tmp = path.with_extension("opening"); + let n = open_file(root, path, &tmp).await?; + tokio::fs::rename(&tmp, path).await?; + Ok(n) +} + +// ─── public specs ──────────────────────────────────────────────────────── + +/// `n` rounded up to a power of two (at least 1 KiB), so sizes leak only +/// their order of magnitude. +pub fn bucket(n: u64) -> u64 { + n.max(1024).next_power_of_two() +} + +/// Spec keys whose (small, scalar) values go in a workload's public spec. +/// Everything else stays sealed: prompts, texts, messages, URLs, headers, +/// input hashes and keys, file names, client keys, tokens. +fn public_keys(workload: &str) -> &'static [&'static str] { + match workload { + "ffmpeg.transcode" => &["preset"], + "tts.synthesize" => &["engine", "voice", "locale", "speed"], + "stt.transcribe" => &["model", "language", "audio_format"], + "diffusion.txt2img" => &["model", "width", "height", "steps"], + "music.txt2music" => &["model", "duration", "steps"], + "video.txt2vid" | "video.img2vid" => &[ + "model", + "image_format", + "frames", + "fps", + "steps", + "width", + "height", + "guidance_scale", + ], + "embed.text" => &["v", "model"], + "ocr.extract" => &["v", "format", "lang"], + "crawl.fetch" => &["v", "timeout_ms"], + "blender.render-frame" => &[ + "v", + "frame_start", + "frame_end", + "engine", + "device", + "format", + "resolution_pct", + "samples", + "frame_timeout_secs", + ], + "workflow.step" => &["v", "kind"], + "infernet.inference" => &["v", "model", "max_tokens", "temperature"], + "infernet.text.infer" => &["base_model", "max_new_tokens"], + "infernet.temporal.infer" => &["model_hash", "ttl_s"], + "webhook.deliver" => &["v", "max_attempts", "timeout_ms", "backoff_ms"], + "gameserver.start" => &[ + "v", + "game", + "hours", + "ram_gb", + "cpus", + "accept_eula", + "backup_mins", + ], + "gameserver.stop" => &["v", "id"], + "captcha.solve" => &["v", "kind"], + _ => &["v"], + } +} + +/// Size fields copied into the public spec as [`bucket`]s. +const SIZE_KEYS: &[&str] = &["audio_size", "image_size", "input_size", "corpus_size"]; + +fn public_scalar(v: &Value) -> bool { + match v { + Value::Bool(_) | Value::Number(_) => true, + Value::String(s) => s.len() <= MAX_PUBLIC_STRING, + _ => false, + } +} + +fn pick(obj: &Map, keys: &[&str]) -> Map { + keys.iter() + .filter_map(|k| { + obj.get(*k) + .filter(|v| public_scalar(v)) + .map(|v| ((*k).to_string(), v.clone())) + }) + .collect() +} + +/// What a worker needs to decide whether to bid on `spec`, and nothing more. +pub fn public_view(workload: &str, spec: &Value) -> Value { + let mut out = Map::new(); + let size = serde_json::to_vec(spec) + .map(|b| b.len() as u64) + .unwrap_or(0); + out.insert("spec_bytes".into(), bucket(size).into()); + let Some(obj) = spec.as_object() else { + return Value::Object(out); + }; + out.extend(pick(obj, public_keys(workload))); + for k in SIZE_KEYS { + if let Some(n) = obj.get(*k).and_then(Value::as_u64) { + out.insert((*k).to_string(), bucket(n).into()); + } + } + match workload { + "ffmpeg.transcode" => { + if let Some(s) = obj.get("spec").and_then(Value::as_object) { + let p = pick( + s, + &[ + "codec", + "bitrate_bps", + "width", + "height", + "keyframe_interval", + "hardware_pref", + ], + ); + out.insert("spec".into(), Value::Object(p)); + } + } + "infernet.text.train" => { + if let Some(j) = obj.get("job").and_then(Value::as_object) { + out.insert( + "job".into(), + Value::Object(pick(j, &["kind", "base_model"])), + ); + } + } + "infernet.temporal.train" => { + if let Some(j) = obj.get("train_job").and_then(Value::as_object) { + let mut p = pick(j, &["schema_version", "kind", "model_type"]); + if let Some(pr) = j.get("privacy").and_then(Value::as_object) { + p.insert( + "privacy".into(), + Value::Object(pick(pr, &["mode", "upload_approved"])), + ); + } + out.insert("train_job".into(), Value::Object(p)); + } + } + "gameserver.stop" => { + if let Some(t) = obj.get("token").and_then(Value::as_str) { + out.insert( + "token_hash".into(), + format!("sha256:{}", hex::encode(Sha256::digest(t.as_bytes()))).into(), + ); + } + } + _ => {} + } + Value::Object(out) +} + +/// A buyer cannot claim one thing publicly and send another: every field it +/// published must equal the field recomputed from the sealed spec. (A subset +/// check, so builds whose public key lists differ still interoperate.) +pub fn public_view_consistent(workload: &str, spec: &Value, published: &Value) -> bool { + let actual = public_view(workload, spec); + fn subset(p: &Value, a: &Value) -> bool { + match (p, a) { + (Value::Object(p), Value::Object(a)) => p + .iter() + .all(|(k, v)| a.get(k).is_some_and(|av| subset(v, av))), + _ => p == a, + } + } + match published { + Value::Null => true, + Value::Object(_) => subset(published, &actual), + _ => false, + } +} + +/// The spec an offer filter should judge: on an e2e offer only the public +/// spec exists at bid time; on a legacy offer the whole spec. +pub enum BidSpec<'a> { + Full(&'a Value), + Public(&'a Value), +} + +pub fn bid_spec(offer: &JobOffer) -> Option> { + match (&offer.e2e, &offer.spec_inline) { + (Some(e), _) => Some(BidSpec::Public(&e.public_spec)), + (None, Some(s)) => Some(BidSpec::Full(s)), + (None, None) => None, + } +} + +/// A public spec with stand-ins for sealed fields that a filter's parser +/// requires (a prompt, a client key), for bid-time checks only: the real +/// spec is checked again by the same filter once it is opened. +pub fn with_placeholders(public: &Value, fill: &[(&str, Value)]) -> Value { + let mut m = public.as_object().cloned().unwrap_or_default(); + for (k, v) in fill { + m.entry((*k).to_string()).or_insert_with(|| v.clone()); + } + Value::Object(m) +} + +/// `offer` as a filter sees it once the sealed spec is open: the full spec in +/// `spec_inline`, no e2e part. Filters run on this again before a won job +/// starts, so every check they make on whole specs still holds. +pub fn opened_offer(offer: &JobOffer, spec: Value) -> JobOffer { + let mut o = offer.clone(); + o.spec_inline = Some(spec); + o.e2e = None; + o +} + +// ─── buyer ─────────────────────────────────────────────────────────────── + +/// The buyer's half of one e2e auction. +pub struct BuyerSession { + secret: StaticSecret, + pub public_hex: String, + commitment: Commitment, + pub attest_nonce: Option, +} + +impl BuyerSession { + pub fn new(spec: &Value, confidential: bool) -> Result { + let secret = StaticSecret::random_from_rng(rand::rngs::OsRng); + let attest_nonce = confidential.then(|| { + let mut n = [0u8; 32]; + rand::rngs::OsRng.fill_bytes(&mut n); + hex::encode(n) + }); + Ok(Self { + public_hex: public_hex(&secret), + secret, + commitment: Commitment::new(spec)?, + attest_nonce, + }) + } + + pub fn spec_hash(&self) -> String { + self.commitment.hash_hex() + } + + pub fn offer_part(&self, workload: &str, spec: &Value) -> OfferE2e { + OfferE2e { + v: JOB_E2E_VERSION, + buyer_key: self.public_hex.clone(), + public_spec: public_view(workload, spec), + attest_nonce: self.attest_nonce.clone(), + } + } + + fn binding<'a>( + &'a self, + job_id: &'a str, + workload: &'a str, + buyer_peer: &'a str, + bid: &'a JobBid, + e: &'a BidE2e, + ) -> Binding<'a> { + Binding { + job_id, + workload, + buyer_peer, + worker_peer: &bid.bidder_peer_id, + buyer_key: &self.public_hex, + worker_key: &e.worker_key, + } + } + + /// A bid is usable only with an X25519 key its bidder signed for this + /// job and buyer key (and, for confidential work, attestation evidence + /// that binds the key and verifies). + pub fn check_bid( + &self, + job_id: &str, + workload: &str, + buyer_peer: &str, + bid: &JobBid, + ) -> Result<()> { + let e = bid.e2e.as_ref().context("bid carries no e2e key")?; + parse_public(&e.worker_key)?; + let peer: PeerId = bid + .bidder_peer_id + .parse() + .map_err(|_| anyhow!("bidder peer id"))?; + let sig = B64 + .decode(&e.sig) + .map_err(|_| anyhow!("bid key signature is not base64"))?; + let stmt = bid_statement(&self.binding(job_id, workload, buyer_peer, bid, e)); + if !c0mpute_net::identity::verify_peer_signature(&peer, &stmt, &sig) { + bail!("bid key signature does not verify against the bidder's peer id"); + } + if let Some(nonce) = &self.attest_nonce { + let evidence = e + .attestation + .as_ref() + .context("confidential job: bid has no attestation")?; + let rd = report_data(job_id, &bid.bidder_peer_id, &e.worker_key, nonce); + attestation_verifier() + .context("confidential job: no attestation verifier is configured on this buyer")? + .verify(evidence, &rd) + .context("attestation does not verify")?; + } + Ok(()) + } + + /// Seal the spec to the winner. Returns the sealed bytes and the key its + /// result will be sealed back with. + pub fn seal_for( + &self, + job_id: &str, + workload: &str, + buyer_peer: &str, + winner: &JobBid, + ) -> Result<(Vec, JobKey)> { + let e = winner.e2e.as_ref().context("winner has no e2e key")?; + let b = self.binding(job_id, workload, buyer_peer, winner, e); + let keys = derive(&self.secret, &e.worker_key, &b)?; + let sealed = seal_spec(&keys, &b, &self.commitment)?; + Ok((sealed, keys.result.clone())) + } +} + +struct PendingPayload { + winner: PeerId, + sealed: Arc>, + until: Instant, +} + +fn payloads() -> &'static Mutex> { + static P: OnceLock>> = OnceLock::new(); + P.get_or_init(Default::default) +} + +/// Removes a job's sealed spec when the auction ends. +pub struct PayloadGuard(String); + +impl Drop for PayloadGuard { + fn drop(&mut self) { + if let Ok(mut p) = payloads().lock() { + p.remove(&self.0); + } + } +} + +/// Hold `sealed` for `winner` to fetch over [`PAYLOAD_SERVICE`]. +pub fn hold_payload( + net: &Libp2pNetwork, + job_id: &str, + winner: PeerId, + sealed: Vec, +) -> PayloadGuard { + net.register_rpc(PAYLOAD_SERVICE, Arc::new(PayloadHandler)); + if let Ok(mut p) = payloads().lock() { + let now = Instant::now(); + p.retain(|_, v| v.until > now); + p.insert( + job_id.to_string(), + PendingPayload { + winner, + sealed: Arc::new(sealed), + until: now + PAYLOAD_TTL, + }, + ); + } + PayloadGuard(job_id.to_string()) +} + +/// Serves sealed specs, each only to the peer that won it. The caller's peer +/// id comes from the noise handshake, so it cannot be claimed falsely. +struct PayloadHandler; + +#[async_trait::async_trait] +impl RpcHandler for PayloadHandler { + async fn handle(&self, peer: PeerId, body: Vec) -> std::result::Result, String> { + let job_id = std::str::from_utf8(&body).map_err(|_| "bad job id".to_string())?; + let p = payloads().lock().map_err(|_| "unavailable".to_string())?; + match p.get(job_id) { + Some(x) if x.winner == peer && x.until > Instant::now() => { + Ok(x.sealed.as_ref().clone()) + } + // Same answer for "no such job" and "not yours". + _ => Err("no sealed spec for this peer".into()), + } + } +} + +// ─── worker ────────────────────────────────────────────────────────────── + +/// The worker's half of one bid: its fresh X25519 secret, kept until the +/// accept (or the bid lapses). +pub struct WorkerSession { + secret: StaticSecret, + pub public_hex: String, +} + +impl std::fmt::Debug for WorkerSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "WorkerSession({})", self.public_hex) + } +} + +impl WorkerSession { + #[cfg(test)] + pub(crate) fn for_tests() -> Self { + let secret = StaticSecret::random_from_rng(rand::rngs::OsRng); + Self { + public_hex: public_hex(&secret), + secret, + } + } + + /// A fresh key for a bid on `offer`, signed by our identity, plus TEE + /// evidence when the offer asks for it. + pub fn for_offer(net: &Libp2pNetwork, offer: &JobOffer) -> Result<(Self, BidE2e)> { + let e = offer.e2e.as_ref().context("offer is not e2e")?; + if e.v != JOB_E2E_VERSION { + bail!("offer speaks e2e v{}, this node v{JOB_E2E_VERSION}", e.v); + } + parse_public(&e.buyer_key)?; + let secret = StaticSecret::random_from_rng(rand::rngs::OsRng); + let public_hex = public_hex(&secret); + let me = net.peer_id().to_base58(); + let b = Binding { + job_id: &offer.job_id, + workload: &offer.workload_type, + buyer_peer: &offer.buyer_peer_id, + worker_peer: &me, + buyer_key: &e.buyer_key, + worker_key: &public_hex, + }; + let sig = B64.encode(net.sign(&bid_statement(&b))?); + let attestation = match &e.attest_nonce { + Some(nonce) => { + let rd = report_data(&offer.job_id, &me, &public_hex, nonce); + Some( + attestation_provider() + .context("no TEE attestation provider on this node")? + .evidence(&rd)?, + ) + } + None => None, + }; + Ok(( + Self { + secret, + public_hex: public_hex.clone(), + }, + BidE2e { + worker_key: public_hex, + sig, + attestation, + }, + )) + } + + /// Fetch the sealed spec from the buyer (point to point), open it, check + /// it against the offer's commitment and public spec, and return the spec + /// with the key the result must be sealed with. + pub async fn open_accept( + &self, + net: &Libp2pNetwork, + offer: &JobOffer, + accept: &JobAccept, + ) -> Result<(Value, JobKey)> { + let e = offer.e2e.as_ref().context("offer is not e2e")?; + let me = net.peer_id().to_base58(); + let b = Binding { + job_id: &offer.job_id, + workload: &offer.workload_type, + buyer_peer: &offer.buyer_peer_id, + worker_peer: &me, + buyer_key: &e.buyer_key, + worker_key: &self.public_hex, + }; + let sealed = fetch_sealed(net, accept).await?; + let keys = derive(&self.secret, &e.buyer_key, &b)?; + let bytes = open_spec(&keys, &b, &sealed, &offer.spec_hash)?; + let spec: Value = serde_json::from_slice(&bytes).context("sealed spec is not JSON")?; + if !public_view_consistent(&offer.workload_type, &spec, &e.public_spec) { + bail!("sealed spec differs from what the offer published"); + } + Ok((spec, keys.result.clone())) + } +} + +async fn fetch_sealed(net: &Libp2pNetwork, accept: &JobAccept) -> Result> { + let buyer: PeerId = accept + .buyer_peer_id + .parse() + .map_err(|_| anyhow!("buyer peer id"))?; + for addr in + crate::reachability::dialable_buyer_addrs(&accept.buyer_addrs, &accept.buyer_peer_id) + { + let mut a = addr; + if matches!(a.iter().last(), Some(c0mpute_net::Protocol::P2p(_))) { + a.pop(); + } + net.add_peer_address(buyer, a).await; + } + let mut last = anyhow!("not tried"); + for attempt in 0..4u32 { + match net + .rpc(buyer, PAYLOAD_SERVICE, accept.job_id.as_bytes().to_vec()) + .await + { + Ok(b) if b.len() <= MAX_SEALED_SPEC => return Ok(b), + Ok(_) => bail!("sealed spec from the buyer is over the limit"), + Err(e) => last = e, + } + tokio::time::sleep(Duration::from_millis(500 * u64::from(attempt + 1))).await; + } + Err(last.context("fetch the sealed spec from the buyer")) +} + +// ─── results ───────────────────────────────────────────────────────────── + +/// The key an e2e job's results are sealed with; `None` for a legacy job. +pub fn result_key(accept: &JobAccept) -> Result> { + match (&accept.e2e, &accept.result_key) { + (Some(_), Some(k)) => Ok(Some(k)), + (Some(_), None) => bail!("e2e job without a result key"), + (None, _) => Ok(None), + } +} + +/// Seal a result for the buyer when the job is e2e; a legacy job's result is +/// stored as it is. +pub fn seal_result(accept: &JobAccept, plain: &[u8]) -> Result>> { + result_key(accept)?.map(|k| seal_blob(k, plain)).transpose() +} + +/// For a result file: the path to store (a sealed copy next to `path` for an +/// e2e job, removed by the caller with [`SealedCopy`]'s drop) and its limit. +pub async fn sealed_result_file( + accept: &JobAccept, + path: &Path, + max_bytes: u64, +) -> Result<(SealedCopy, u64)> { + match result_key(accept)? { + Some(k) => { + let sealed = path.with_extension("sealed"); + seal_file(k, path, &sealed).await?; + Ok(( + SealedCopy { + path: sealed, + temp: true, + }, + sealed_len(max_bytes), + )) + } + None => Ok(( + SealedCopy { + path: path.to_path_buf(), + temp: false, + }, + max_bytes, + )), + } +} + +/// A file to store; removed on drop when it is a sealed temporary copy. +pub struct SealedCopy { + pub path: std::path::PathBuf, + temp: bool, +} + +impl Drop for SealedCopy { + fn drop(&mut self) { + if self.temp { + let _ = std::fs::remove_file(&self.path); + } + } +} + +// ─── confidential computing (TEE attestation) ──────────────────────────── + +/// 64 bytes a TEE report must carry as its report data / REPORTDATA: binds +/// the attestation to this job, this worker and its X25519 key, and the +/// buyer's fresh nonce. +pub fn report_data(job_id: &str, worker_peer: &str, worker_key: &str, nonce: &str) -> [u8; 64] { + let mut h = blake3::Hasher::new_derive_key("c0mpute/job-e2e/v1 attestation report data"); + for f in [job_id, worker_peer, worker_key, nonce] { + h.update(&(f.len() as u32).to_be_bytes()); + h.update(f.as_bytes()); + } + let mut out = [0u8; 64]; + h.finalize_xof().fill(&mut out); + out +} + +/// Buyer side: checks a worker's attestation evidence (an AMD SEV-SNP report +/// with its VCEK chain, an Intel TDX quote...) against vendor roots and the +/// buyer's measurement policy, and that its report data equals +/// `report_data`. Only then is the spec sealed to the attested key. +pub trait AttestationVerifier: Send + Sync { + fn verify(&self, evidence: &Value, report_data: &[u8; 64]) -> Result<()>; +} + +/// Worker side: produces attestation evidence over `report_data` from the +/// TEE this node runs in. +pub trait AttestationProvider: Send + Sync { + fn evidence(&self, report_data: &[u8; 64]) -> Result; +} + +static VERIFIER: RwLock>> = RwLock::new(None); +static PROVIDER: RwLock>> = RwLock::new(None); + +pub fn set_attestation_verifier(v: Option>) { + if let Ok(mut g) = VERIFIER.write() { + *g = v; + } +} + +pub fn set_attestation_provider(p: Option>) { + if let Ok(mut g) = PROVIDER.write() { + *g = p; + } +} + +pub fn attestation_verifier() -> Option> { + VERIFIER.read().ok().and_then(|g| g.clone()) +} + +pub fn attestation_provider() -> Option> { + PROVIDER.read().ok().and_then(|g| g.clone()) +} + +/// Capability tags a worker satisfies by speaking this protocol rather than +/// by configuration: [`JOB_E2E_TAG`] always, [`CONFIDENTIAL_TAG`] only with +/// an attestation provider. +pub fn implicit_tag(tag: &str) -> bool { + tag == JOB_E2E_TAG || (tag == CONFIDENTIAL_TAG && attestation_provider().is_some()) +} + +/// Whether this build still serves plaintext (pre-e2e) offers. The window +/// closes in the release after the one that introduced e2e jobs; +/// `C0MPUTE_REJECT_PLAINTEXT_OFFERS=1` closes it early on one node. +pub fn serves_plaintext_offers() -> bool { + !std::env::var("C0MPUTE_REJECT_PLAINTEXT_OFFERS").is_ok_and(|v| !v.is_empty() && v != "0") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn binding<'a>(bk: &'a str, wk: &'a str) -> Binding<'a> { + Binding { + job_id: "job-1", + workload: "infernet.inference", + buyer_peer: "B", + worker_peer: "W", + buyer_key: bk, + worker_key: wk, + } + } + + #[test] + fn both_sides_derive_the_same_keys_and_bindings_separate_them() { + let b = StaticSecret::random_from_rng(rand::rngs::OsRng); + let w = StaticSecret::random_from_rng(rand::rngs::OsRng); + let (bp, wp) = (public_hex(&b), public_hex(&w)); + let kb = derive(&b, &wp, &binding(&bp, &wp)).unwrap(); + let kw = derive(&w, &bp, &binding(&bp, &wp)).unwrap(); + assert_eq!(kb.spec, kw.spec); + assert_eq!(kb.result, kw.result); + assert_ne!(kb.spec, kb.result); + let mut other = binding(&bp, &wp); + other.job_id = "job-2"; + assert_ne!(derive(&b, &wp, &other).unwrap().spec, kb.spec); + } + + #[test] + fn low_order_keys_are_refused() { + let b = StaticSecret::random_from_rng(rand::rngs::OsRng); + assert!(derive(&b, &"00".repeat(32), &binding("x", "y")).is_err()); + } + + #[test] + fn sealed_spec_opens_only_for_the_winner_and_only_unmodified() { + let spec = json!({"v": 1, "messages": [{"role": "user", "content": "the secret prompt"}]}); + let c = Commitment::new(&spec).unwrap(); + let b = StaticSecret::random_from_rng(rand::rngs::OsRng); + let w = StaticSecret::random_from_rng(rand::rngs::OsRng); + let loser = StaticSecret::random_from_rng(rand::rngs::OsRng); + let (bp, wp, lp_) = (public_hex(&b), public_hex(&w), public_hex(&loser)); + let bind = binding(&bp, &wp); + let sealed = seal_spec(&derive(&b, &wp, &bind).unwrap(), &bind, &c).unwrap(); + assert!(!String::from_utf8_lossy(&sealed).contains("secret prompt")); + let opened = open_spec( + &derive(&w, &bp, &bind).unwrap(), + &bind, + &sealed, + &c.hash_hex(), + ) + .unwrap(); + assert_eq!(serde_json::from_slice::(&opened).unwrap(), spec); + // The loser (or anyone else) derives a different key. + let lbind = binding(&bp, &lp_); + assert!( + open_spec( + &derive(&loser, &bp, &lbind).unwrap(), + &lbind, + &sealed, + &c.hash_hex() + ) + .is_err() + ); + assert!( + open_spec( + &derive(&loser, &bp, &bind).unwrap(), + &bind, + &sealed, + &c.hash_hex() + ) + .is_err() + ); + // Tampering anywhere fails. + for i in [0, 12, sealed.len() - 1] { + let mut t = sealed.clone(); + t[i] ^= 1; + assert!(open_spec(&derive(&w, &bp, &bind).unwrap(), &bind, &t, &c.hash_hex()).is_err()); + } + // Moved to another job: the associated data no longer matches. + let mut moved = bind; + moved.job_id = "job-2"; + assert!( + open_spec( + &derive(&w, &bp, &bind).unwrap(), + &moved, + &sealed, + &c.hash_hex() + ) + .is_err() + ); + // A different commitment is refused even with the right key. + let other = Commitment::new(&spec).unwrap(); + assert!( + open_spec( + &derive(&w, &bp, &bind).unwrap(), + &bind, + &sealed, + &other.hash_hex() + ) + .is_err() + ); + } + + #[test] + fn commitment_hides_the_spec() { + let spec = json!({"text": "hi"}); + let a = Commitment::new(&spec).unwrap(); + let b = Commitment::new(&spec).unwrap(); + assert_ne!( + a.hash_hex(), + b.hash_hex(), + "salted: equal specs commit differently" + ); + assert_ne!( + a.hash_hex(), + blake3::hash(spec.to_string().as_bytes()) + .to_hex() + .to_string() + ); + } + + #[test] + fn blobs_round_trip_at_segment_edges_and_reject_tampering() { + let k = random_key(); + for n in [ + 0, + 1, + BLOB_SEGMENT - 1, + BLOB_SEGMENT, + BLOB_SEGMENT + 1, + 3 * BLOB_SEGMENT, + ] { + let plain: Vec = (0..n).map(|i| (i % 251) as u8).collect(); + let sealed = seal_blob(&k, &plain).unwrap(); + assert_eq!(sealed.len() as u64, sealed_len(n as u64), "n={n}"); + assert_eq!(open_blob(&k, &sealed).unwrap(), plain, "n={n}"); + assert!(open_blob(&random_key(), &sealed).is_err()); + if n > BLOB_SEGMENT { + // Drop the final segment: the new last one lacks the flag. + let cut = BLOB_HEADER + (BLOB_SEGMENT + TAG_LEN); + assert!(open_blob(&k, &sealed[..cut]).is_err(), "truncation n={n}"); + } + let mut t = sealed.clone(); + let last = t.len() - 1; + t[last] ^= 0x80; + assert!(open_blob(&k, &t).is_err()); + } + // Same plaintext, same root: different ciphertext (fresh salt). + assert_ne!(seal_blob(&k, b"x").unwrap(), seal_blob(&k, b"x").unwrap()); + } + + #[tokio::test] + async fn files_seal_and_open_streamed() { + let dir = tempfile::tempdir().unwrap(); + let k = random_key(); + for n in [0usize, 10, BLOB_SEGMENT, 2 * BLOB_SEGMENT + 7] { + let src = dir.path().join("in"); + let plain: Vec = (0..n).map(|i| (i * 7 % 256) as u8).collect(); + std::fs::write(&src, &plain).unwrap(); + let sealed = dir.path().join("sealed"); + let size = seal_file(&k, &src, &sealed).await.unwrap(); + assert_eq!(size, sealed_len(n as u64)); + // File and memory formats are the same. + assert_eq!( + open_blob(&k, &std::fs::read(&sealed).unwrap()).unwrap(), + plain + ); + let out = dir.path().join("out"); + assert_eq!(open_file(&k, &sealed, &out).await.unwrap(), n as u64); + assert_eq!(std::fs::read(&out).unwrap(), plain); + let mut bytes = std::fs::read(&sealed).unwrap(); + let mid = bytes.len() / 2; + bytes[mid] ^= 1; + std::fs::write(&sealed, &bytes).unwrap(); + assert!(open_file(&k, &sealed, &out).await.is_err()); + assert!(!out.exists(), "a failed open leaves no partial plaintext"); + } + } + + #[test] + fn public_views_carry_no_content() { + let cases = [ + ( + "infernet.inference", + json!({"v": 1, "model": "qwen2.5:0.5b", "messages": [{"role": "user", "content": "SECRET-1"}], "max_tokens": 64}), + ), + ( + "tts.synthesize", + json!({"text": "SECRET-1", "engine": "kokoro", "voice": "af_heart", "locale": "en-US", "speed": 1.0, "persona": "SECRET-1"}), + ), + ( + "stt.transcribe", + json!({"audio_hash": "SECRET-1", "audio_size": 12345, "audio_pieces": ["SECRET-1"], "audio_key": "SECRET-1", "model": "base", "audio_format": "wav"}), + ), + ( + "diffusion.txt2img", + json!({"prompt": "SECRET-1", "model": "sd", "width": 512, "height": 512, "steps": 20, "seed": 7}), + ), + ( + "music.txt2music", + json!({"prompt": "SECRET-1", "lyrics": "SECRET-1", "duration": 30}), + ), + ( + "video.img2vid", + json!({"model": "m", "prompt": "SECRET-1", "negative_prompt": "SECRET-1", "image_hash": "SECRET-1", "image_key": "SECRET-1", "frames": 16}), + ), + ( + "embed.text", + json!({"v": 1, "model": "e5", "texts": ["SECRET-1"]}), + ), + ( + "ocr.extract", + json!({"v": 1, "input_hash": "SECRET-1", "input_key": "SECRET-1", "format": "png", "lang": "eng"}), + ), + ( + "crawl.fetch", + json!({"v": 1, "url": "https://SECRET-1.example/", "headers": {"cookie": "SECRET-1"}}), + ), + ( + "blender.render-frame", + json!({"v": 1, "input_hash": "SECRET-1", "input_size": 9, "input_pieces": ["SECRET-1"], "frame_start": 1, "frame_end": 2}), + ), + ( + "workflow.step", + json!({"v": 1, "kind": "http_call", "config": {"url": "https://SECRET-1/"}, "input": "SECRET-1"}), + ), + ( + "ffmpeg.transcode", + json!({"input_url": "https://SECRET-1/a.mp4", "preset": "720p", "spec": {"codec": "h264", "bitrate_bps": 1, "width": 1, "height": 1, "keyframe_interval": 1, "extra_ffmpeg_args": ["drawtext=SECRET-1"]}}), + ), + ( + "infernet.text.train", + json!({"job": {"kind": "lora", "base_model": "org/m", "data": "SECRET-1"}, "dataset_hash": "SECRET-1", "dataset_url": "https://SECRET-1"}), + ), + ( + "infernet.text.infer", + json!({"adapter_hash": "SECRET-1", "base_model": "org/m", "messages": [{"role": "user", "content": "SECRET-1"}], "max_new_tokens": 9}), + ), + ( + "gameserver.stop", + json!({"v": 1, "id": "srv", "token": "SECRET-1"}), + ), + ( + "gameserver.start", + json!({"v": 1, "game": "minecraft", "hours": 1, "ram_gb": 2, "cpus": 1.0, "stop_hash": "SECRET-1", "client_pubkey": "SECRET-1", "backup_pubkey": "SECRET-1", "world": "SECRET-1", "server_name": "SECRET-1"}), + ), + ( + "captcha.solve", + json!({"v": 1, "kind": "text", "image": "SECRET-1"}), + ), + ( + "webhook.deliver", + json!({"v": 1, "client_pubkey": "SECRET-1", "max_attempts": 3, "timeout_ms": 1000, "backoff_ms": 100}), + ), + ( + "infernet.temporal.infer", + json!({"model_hash": "sha256:00", "session_id": "SECRET-1", "ttl_s": 60, "account_scope": "SECRET-1", "client_pubkey": "SECRET-1"}), + ), + ( + "infernet.temporal.train", + json!({"train_job": {"kind": "train_job", "dataset_hash": "SECRET-1", "privacy": {"mode": "trusted_worker"}}, "dataset_url": "https://SECRET-1", "dataset_hash": "SECRET-1"}), + ), + ("unknown.workload", json!({"v": 1, "anything": "SECRET-1"})), + ]; + for (w, spec) in cases { + let v = public_view(w, &spec); + assert!(!v.to_string().contains("SECRET-1"), "{w}: {v}"); + assert!(public_view_consistent(w, &spec, &v), "{w}"); + } + let v = public_view("infernet.inference", &cases_model()); + assert_eq!(v["model"], "qwen2.5:0.5b"); + assert_eq!( + public_view("stt.transcribe", &json!({"audio_size": 12345}))["audio_size"], + 16384 + ); + } + + fn cases_model() -> Value { + json!({"v": 1, "model": "qwen2.5:0.5b", "messages": []}) + } + + #[test] + fn a_buyer_cannot_publish_one_spec_and_seal_another() { + let spec = json!({"v": 1, "model": "big-model", "messages": []}); + let claimed = public_view( + "infernet.inference", + &json!({"v": 1, "model": "small-model", "messages": []}), + ); + assert!(!public_view_consistent( + "infernet.inference", + &spec, + &claimed + )); + assert!(public_view_consistent( + "infernet.inference", + &spec, + &json!({}) + )); + assert!(!public_view_consistent( + "infernet.inference", + &spec, + &json!("x") + )); + } + + #[test] + fn report_data_binds_every_field() { + let a = report_data("j", "w", "k", "n"); + assert_ne!(a, report_data("j", "w", "k", "m")); + assert_ne!(a, report_data("j", "w", "kk", "n")); + assert_ne!( + report_data("ab", "c", "k", "n"), + report_data("a", "bc", "k", "n") + ); + } +} diff --git a/node/crates/c0mpute-core/src/jobs_gateway.rs b/node/crates/c0mpute-core/src/jobs_gateway.rs index e4c430d1..5c03e08c 100644 --- a/node/crates/c0mpute-core/src/jobs_gateway.rs +++ b/node/crates/c0mpute-core/src/jobs_gateway.rs @@ -24,7 +24,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use c0mpute_net::topics::JobStatus; -use c0mpute_net::{Libp2pNetwork, Network}; +use c0mpute_net::Libp2pNetwork; use c0mpute_store::ChunkStore; use serde::Deserialize; use serde_json::{Value, json}; @@ -151,6 +151,9 @@ impl Workload { #[derive(Clone)] pub struct JobsBridge { net: Arc, + // Specs used to be served from here by hash; sealed specs (DIP-0032) + // go point to point instead. Kept for the bridge's own inputs. + #[allow(dead_code)] store: ChunkStore, token: Arc, max_price_usd: f64, @@ -459,15 +462,11 @@ async fn run_job( let required = workload.required_capabilities(&spec); // Kept for checking the result (embeddings: one vector per text). let spec_for_check = spec.clone(); - // Small specs ride in the gossip offer; large ones are served from this - // node's store and fetched by the winner by hash. - let spec_inline = if spec_bytes.len() <= workflow::MAX_INLINE_SPEC_BYTES { - spec - } else { - let hash = state.store.put(&spec_bytes).await.map_err(internal)?; - state.net.announce(&hash).await.map_err(internal)?; - workflow::spec_ref(&hash, spec_bytes.len()) - }; + // The spec never rides the gossip offer: it is sealed to the winner and + // fetched point to point (DIP-0032), whatever its size, so it is never + // stored here as a plaintext chunk anyone could fetch by hash. + let _ = spec_bytes; + let spec_inline = spec; let remaining = timeout.saturating_sub(started.elapsed()); let auction = JobAuction::new(workload.name(), spec_inline) @@ -508,8 +507,7 @@ async fn run_job( .min(Duration::from_secs(10 * 60)); let bytes = match fetch_result( &state.net, - &output_hash, - &worker, + &outcome, workload.max_result_bytes(), fetch_budget, ) @@ -534,7 +532,7 @@ async fn run_job( } let result = if workload == Workload::RenderFrame { let budget = timeout.saturating_sub(started.elapsed()); - attach_render_frames(state, &spec_for_check, result, &worker, budget) + attach_render_frames(state, &spec_for_check, result, &worker, &outcome.result_key, budget) .await .map_err(|error| worker_failed(&format!("{error:#}")))? } else { @@ -559,6 +557,7 @@ async fn attach_render_frames( spec: &Value, mut result: Value, worker: &str, + key: &c0mpute_net::topics::JobKey, budget: Duration, ) -> Result { use base64::Engine; @@ -568,7 +567,7 @@ async fn attach_render_frames( if total > MAX_INLINE_FRAME_BYTES { bail!("frames total {total} bytes, over the bridge's {MAX_INLINE_FRAME_BYTES}-byte limit; use `c0mpute render`"); } - let frames = crate::render::fetch_frames(&state.net, worker, &metas, budget).await?; + let frames = crate::render::fetch_frames(&state.net, worker, &metas, key, budget).await?; if let Some(list) = result.get_mut("frames").and_then(Value::as_array_mut) { for (entry, (_, bytes)) in list.iter_mut().zip(frames) { entry["data_base64"] = @@ -578,14 +577,6 @@ async fn attach_render_frames( Ok(result) } -fn internal(error: anyhow::Error) -> ApiError { - tracing::warn!(err = %error, "jobs bridge operation failed"); - ApiError( - StatusCode::INTERNAL_SERVER_ERROR, - json!({ "error": "internal", "detail": "jobs bridge operation failed" }), - ) -} - #[cfg(test)] mod tests { use super::*; diff --git a/node/crates/c0mpute-core/src/lib.rs b/node/crates/c0mpute-core/src/lib.rs index e35aa8c2..4f65b88b 100644 --- a/node/crates/c0mpute-core/src/lib.rs +++ b/node/crates/c0mpute-core/src/lib.rs @@ -21,6 +21,7 @@ pub mod crowdgpt; pub mod diffusion; pub mod dm; pub mod music; +pub mod job_e2e; pub mod job_progress; pub mod job_process; pub mod hosting; diff --git a/node/crates/c0mpute-core/src/music.rs b/node/crates/c0mpute-core/src/music.rs index 643242bd..c396e804 100644 --- a/node/crates/c0mpute-core/src/music.rs +++ b/node/crates/c0mpute-core/src/music.rs @@ -701,7 +701,7 @@ async fn run_job( bail!("ACE-Step output is not MP3/FLAC audio under the 48 MiB result limit"); } // Over 1 MiB goes back as pieces: one chunk fetch caps out near 5 MiB. - crate::pieces::publish_result_bytes(net, store, &bytes).await + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes).await } #[cfg(test)] diff --git a/node/crates/c0mpute-core/src/ocr.rs b/node/crates/c0mpute-core/src/ocr.rs index 7f30f8c5..e36c2536 100644 --- a/node/crates/c0mpute-core/src/ocr.rs +++ b/node/crates/c0mpute-core/src/ocr.rs @@ -68,6 +68,10 @@ pub struct OcrSpec { /// looks the hash up on the DHT. #[serde(default)] pub input_provider_peer_id: Option, + /// Hex key the stored document is sealed under (DIP-0032); rides only inside + /// the sealed spec. Absent: the stored bytes are plaintext. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_key: Option, /// One of [`FORMATS`]; the worker re-detects it from the bytes. pub format: String, /// Tesseract language(s), e.g. `eng` or `eng+deu`. @@ -87,7 +91,7 @@ impl OcrSpec { &self.input_hash, self.input_size, self.input_pieces.as_deref(), - MAX_INPUT_BYTES, + crate::pieces::input_limit(MAX_INPUT_BYTES, self.input_key.is_some()), ) } } @@ -262,12 +266,14 @@ pub fn parse_lang_list(text: &str) -> Vec { /// Bid filter: skip offers this box cannot complete (a language it lacks, a /// PDF without poppler, or a spec it cannot read). pub fn offer_filter() -> crate::dispatch::OfferFilter { - Arc::new(|offer: &JobOffer| { - offer - .spec_inline - .as_ref() - .and_then(|spec| parse_spec(spec).ok()) - .is_some_and(|spec| runtime().can_serve(&spec)) + Arc::new(|offer: &JobOffer| match crate::job_e2e::bid_spec(offer) { + Some(crate::job_e2e::BidSpec::Public(v)) => { + // The input stays sealed: judge the format and language. + let spec = crate::job_e2e::with_placeholders(v, &[("input_hash", serde_json::json!("0".repeat(64)))]); + parse_spec(&spec).is_ok_and(|spec| runtime().can_serve(&spec)) + } + Some(crate::job_e2e::BidSpec::Full(v)) => parse_spec(v).is_ok_and(|spec| runtime().can_serve(&spec)), + None => false, }) } @@ -481,8 +487,9 @@ pub async fn fetch_input(net: &Arc, spec: &OcrSpec) -> Result MAX_RESULT_BYTES { bail!("OCR result is over the {MAX_RESULT_BYTES}-byte limit"); } - crate::pieces::publish_result_bytes(net, store, &bytes) + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes) .await .context("publish OCR result") } @@ -624,6 +631,7 @@ mod tests { pdf: false, }; let spec = |format: &str, lang: &str| OcrSpec { + input_key: None, v: 1, input_hash: hash(), input_size: None, diff --git a/node/crates/c0mpute-core/src/pieces.rs b/node/crates/c0mpute-core/src/pieces.rs index 64cbdc3d..93012690 100644 --- a/node/crates/c0mpute-core/src/pieces.rs +++ b/node/crates/c0mpute-core/src/pieces.rs @@ -139,7 +139,65 @@ pub async fn put_bytes_pieces( }) } +/// Seal `bytes` under a fresh key ([`crate::job_e2e`]) and store the +/// ciphertext as pieces. The key belongs in the sealed spec only +/// ([`Pieces::sealed_spec_fields`]): every node that stores, relays or fetches +/// the pieces sees ciphertext. +pub async fn put_bytes_pieces_sealed( + net: &Libp2pNetwork, + store: &ChunkStore, + bytes: &[u8], + piece_bytes: usize, +) -> Result<(Pieces, c0mpute_net::topics::JobKey)> { + if bytes.is_empty() { + bail!("cannot store an empty input"); + } + let key = crate::job_e2e::random_key(); + let sealed = crate::job_e2e::seal_blob(&key, bytes)?; + Ok((put_bytes_pieces(net, store, &sealed, piece_bytes).await?, key)) +} + +/// [`put_bytes_pieces_sealed`] for a file on disk, streamed through a sealed +/// copy next to it (removed afterwards). +pub async fn put_file_pieces_sealed( + net: &Libp2pNetwork, + store: &ChunkStore, + path: &Path, + max_bytes: u64, + piece_bytes: usize, +) -> Result<(Pieces, c0mpute_net::topics::JobKey)> { + let size = tokio::fs::metadata(path).await.with_context(|| format!("read {}", path.display()))?.len(); + if size == 0 || size > max_bytes { + bail!("{} is {size} bytes; the limit is {max_bytes}", path.display()); + } + let key = crate::job_e2e::random_key(); + let sealed = path.with_extension(format!("sealed-{}", uuid::Uuid::new_v4())); + crate::job_e2e::seal_file(&key, path, &sealed).await?; + let stored = put_file_pieces(net, store, &sealed, crate::job_e2e::sealed_len(max_bytes), piece_bytes).await; + let _ = tokio::fs::remove_file(&sealed).await; + Ok((stored?, key)) +} + +/// The byte limit to check a stored input against: its plaintext limit, or +/// that limit once sealed when the spec names a key. +pub fn input_limit(max_bytes: usize, sealed: bool) -> usize { + if sealed { crate::job_e2e::sealed_len(max_bytes as u64) as usize } else { max_bytes } +} + +/// The key a spec's `_key` field names, if any. +pub fn input_key(hex_key: Option<&str>) -> Result> { + hex_key.map(crate::job_e2e::key_from_hex).transpose() +} + impl Pieces { + /// [`Self::spec_fields`] plus `_key`, the key the stored input + /// is sealed under. Only ever inside a sealed spec. + pub fn sealed_spec_fields(&self, prefix: &str, key: &c0mpute_net::topics::JobKey) -> serde_json::Map { + let mut map = self.spec_fields(prefix); + map.insert(format!("{prefix}_key"), crate::job_e2e::key_to_hex(key).into()); + map + } + /// The spec fields that name this input, under `prefix` (`input`, /// `audio`, `image`): `_hash` always, plus `_size` and /// `_pieces` when there is more than one piece. A one-piece @@ -384,6 +442,55 @@ impl InputRef { } } + /// [`Self::fetch`], then open with `key` when the input is sealed. The + /// stored bytes are checked against the sealed size of `max_bytes`, the + /// opened bytes against `max_bytes`. + pub async fn fetch_opened( + &self, + net: &Arc, + provider: Option, + max_bytes: usize, + key: Option<&c0mpute_net::topics::JobKey>, + ) -> Result> { + let Some(key) = key else { + return self.fetch(net, provider, max_bytes).await; + }; + let sealed = self.fetch(net, provider, input_limit(max_bytes, true)).await?; + let plain = crate::job_e2e::open_blob(key, &sealed).context("the input does not open with its key")?; + if plain.is_empty() || plain.len() > max_bytes { + bail!("the input is empty or over the {max_bytes}-byte limit"); + } + Ok(plain) + } + + /// [`Self::fetch_to_file`], then open into `dest` with `key` when the + /// input is sealed (the sealed copy is removed). + pub async fn fetch_to_file_opened( + &self, + net: &Arc, + provider: Option, + max_bytes: usize, + key: Option<&c0mpute_net::topics::JobKey>, + dest: &Path, + ) -> Result<()> { + let Some(key) = key else { + return self.fetch_to_file(net, provider, max_bytes, dest).await; + }; + let sealed = dest.with_extension(format!("sealed-{}", uuid::Uuid::new_v4())); + let fetched = self.fetch_to_file(net, provider, input_limit(max_bytes, true), &sealed).await; + let opened = match fetched { + Ok(()) => crate::job_e2e::open_file(key, &sealed, dest).await.context("the input does not open with its key"), + Err(e) => Err(e), + }; + let _ = tokio::fs::remove_file(&sealed).await; + let n = opened?; + if n as usize > max_bytes { + let _ = tokio::fs::remove_file(dest).await; + bail!("the input is over the {max_bytes}-byte limit"); + } + Ok(()) + } + /// Fetch and verify the whole input into memory. pub async fn fetch( &self, @@ -570,11 +677,27 @@ async fn store_manifest( /// receipt names. Up to [`RESULT_SINGLE_CHUNK_MAX`] it is the result's own /// hash (one chunk, readable by every buyer); above that it is a manifest /// over [`RESULT_PIECE_BYTES`] pieces. +/// +/// An e2e job's result ([`crate::job_e2e`]) is sealed to the buyer first, so +/// what is stored, announced and named by the receipt is ciphertext. pub async fn publish_result_bytes( net: &Libp2pNetwork, store: &ChunkStore, + accept: &c0mpute_net::topics::JobAccept, bytes: &[u8], ) -> Result { + if bytes.is_empty() { + bail!("cannot publish an empty result"); + } + match crate::job_e2e::seal_result(accept, bytes)? { + Some(sealed) => publish_stored_bytes(net, store, &sealed).await, + None => publish_stored_bytes(net, store, bytes).await, + } +} + +/// Store and announce `bytes` exactly as given (no sealing). Runners use +/// [`publish_result_bytes`], which seals e2e results first. +pub async fn publish_stored_bytes(net: &Libp2pNetwork, store: &ChunkStore, bytes: &[u8]) -> Result { if bytes.is_empty() { bail!("cannot publish an empty result"); } @@ -603,9 +726,33 @@ pub async fn publish_result_bytes( pub async fn publish_result_file( net: &Libp2pNetwork, store: &ChunkStore, + accept: &c0mpute_net::topics::JobAccept, path: &Path, max_bytes: u64, ) -> Result { + let size = tokio::fs::metadata(path) + .await + .with_context(|| format!("result file {} is missing", path.display()))? + .len(); + if size == 0 { + bail!("result {} is empty", path.display()); + } + if size > max_bytes { + bail!("result is {size} bytes, over the {max_bytes}-byte result limit"); + } + if accept.e2e.is_none() { + return publish_stored_file(net, store, path, max_bytes).await; + } + let key = accept.result_key.as_ref().context("e2e job without a result key")?; + let sealed = path.with_extension("sealed"); + crate::job_e2e::seal_file(key, path, &sealed).await?; + let out = publish_stored_file(net, store, &sealed, crate::job_e2e::sealed_len(max_bytes)).await; + let _ = tokio::fs::remove_file(&sealed).await; + out +} + +/// [`publish_stored_bytes`] for a file on disk, streamed. +pub async fn publish_stored_file(net: &Libp2pNetwork, store: &ChunkStore, path: &Path, max_bytes: u64) -> Result { let size = tokio::fs::metadata(path) .await .with_context(|| format!("result file {} is missing", path.display()))? @@ -618,7 +765,7 @@ pub async fn publish_result_file( } if size as usize <= RESULT_SINGLE_CHUNK_MAX { let bytes = tokio::fs::read(path).await?; - return publish_result_bytes(net, store, &bytes).await; + return publish_stored_bytes(net, store, &bytes).await; } let pieces = put_file_pieces(net, store, path, max_bytes, RESULT_PIECE_BYTES).await?; store_manifest(net, store, &pieces).await diff --git a/node/crates/c0mpute-core/src/render.rs b/node/crates/c0mpute-core/src/render.rs index b315c591..ec756161 100644 --- a/node/crates/c0mpute-core/src/render.rs +++ b/node/crates/c0mpute-core/src/render.rs @@ -201,6 +201,10 @@ pub struct RenderSpec { /// worker looks each piece up on the DHT. #[serde(default)] pub input_provider_peer_id: Option, + /// Hex key the stored `.blend` is sealed under (DIP-0032); rides only + /// inside the sealed spec. Absent: the stored bytes are plaintext. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_key: Option, pub frame_start: u32, pub frame_end: u32, #[serde(default = "default_engine")] @@ -265,9 +269,14 @@ pub fn parse_spec(spec: &Value) -> std::result::Result { } let input_hash = Hash::from_hex(&s.input_hash) .map_err(|_| "input_hash must be a hex content hash".to_string())?; - if s.input_size == 0 || s.input_size > MAX_INPUT_BYTES { + // A sealed .blend is a little bigger than the plaintext limit. + let max_input = if s.input_key.is_some() { crate::job_e2e::sealed_len(MAX_INPUT_BYTES) } else { MAX_INPUT_BYTES }; + if s.input_size == 0 || s.input_size > max_input { return Err("input_size must be between 1 byte and 512 MiB".into()); } + if s.input_key.as_deref().is_some_and(|k| crate::job_e2e::key_from_hex(k).is_err()) { + return Err("input_key must be 32 bytes of hex".into()); + } let expected = piece_count(s.input_size); if s.input_pieces.len() != expected { return Err(format!( @@ -561,7 +570,8 @@ pub fn result_frames(parsed: &Parsed, value: &Value) -> Result> { parsed.format.name() ); } - if meta.size == 0 || meta.size > MAX_FRAME_BYTES || meta.width == 0 || meta.height == 0 { + // `size` is the sealed frame's: a little over the plaintext cap. + if meta.size == 0 || meta.size > crate::job_e2e::sealed_len(MAX_FRAME_BYTES) || meta.width == 0 || meta.height == 0 { bail!("frame {} has an invalid size or dimensions", meta.frame); } if meta.pieces.len() != piece_count(meta.size) { @@ -590,6 +600,12 @@ pub fn verify_frame(meta: &FrameMeta, bytes: &[u8]) -> Result { if Hash::of(bytes).to_hex() != meta.hash { bail!("frame {} does not match its hash", meta.frame); } + check_frame_image(meta, bytes) +} + +/// The image checks of [`verify_frame`]: a PNG, JPEG or EXR whose header +/// matches the result's format and dimensions. +pub fn check_frame_image(meta: &FrameMeta, bytes: &[u8]) -> Result { let info = inspect_image(bytes) .with_context(|| format!("frame {} is not a valid PNG, JPEG or EXR", meta.frame))?; if info.format.name() != meta.format || info.width != meta.width || info.height != meta.height { @@ -631,8 +647,11 @@ impl Runtime { self.gpu_backends.first().map(String::as_str) } pub fn can_serve(&self, parsed: &Parsed) -> bool { + self.can_serve_engine(parsed.engine, parsed.device) + } + pub fn can_serve_engine(&self, engine: Engine, device: Device) -> bool { self.ready() - && match (parsed.engine.needs_gpu(), parsed.device) { + && match (engine.needs_gpu(), device) { (true, _) | (false, Device::Gpu) => self.gpu().is_some(), (false, _) => true, } @@ -787,12 +806,17 @@ pub fn parse_gpu_probe(text: &str) -> Vec { /// Bid filter: skip offers this box cannot render (a GPU engine or device /// without a GPU, or a spec it cannot read). pub fn offer_filter() -> crate::dispatch::OfferFilter { - Arc::new(|offer: &JobOffer| { - offer - .spec_inline - .as_ref() - .and_then(|spec| parse_spec(spec).ok()) - .is_some_and(|parsed| runtime().can_serve(&parsed)) + Arc::new(|offer: &JobOffer| match crate::job_e2e::bid_spec(offer) { + // The .blend file stays sealed: judge the engine and device. + Some(crate::job_e2e::BidSpec::Public(v)) => { + let field = |k: &str, d: &str| v.get(k).and_then(Value::as_str).unwrap_or(d).to_string(); + match (Engine::parse(&field("engine", &default_engine())), Device::parse(&field("device", &default_device()))) { + (Some(engine), Some(device)) => runtime().can_serve_engine(engine, device), + _ => false, + } + } + Some(crate::job_e2e::BidSpec::Full(spec)) => parse_spec(spec).is_ok_and(|parsed| runtime().can_serve(&parsed)), + None => false, }) } @@ -1137,14 +1161,26 @@ async fn run_render_job( ) -> Result { let spec = crate::workflow::resolve_spec(net, job).await?; let parsed = parse_spec(&spec).map_err(anyhow::Error::msg)?; - let blend = ensure_input(net, &cache_root.join("render-inputs"), &parsed).await?; + let cached = ensure_input(net, &cache_root.join("render-inputs"), &parsed).await?; let job_dir = safe_job_dir(cache_root, &job.accept.job_id)?; tokio::fs::create_dir_all(&job_dir).await?; + // A sealed .blend stays sealed in the cache; the plaintext lives only in + // this job's directory, removed when the job ends. + let blend = match crate::pieces::input_key(parsed.spec.input_key.as_deref())? { + Some(key) => { + let plain = job_dir.join("input.blend"); + crate::job_e2e::open_file(&key, &cached, &plain).await.context("the .blend does not open with its key")?; + plain + } + None => cached, + }; let frames = render(&blend, &parsed, &job_dir).await?; let rt = runtime(); let mut metas = Vec::with_capacity(frames.len()); for f in &frames { - let stored = put_file_pieces(net, store, &f.path, MAX_FRAME_BYTES).await?; + // An e2e job's frames are sealed to the buyer before they are stored. + let (file, limit) = crate::job_e2e::sealed_result_file(&job.accept, &f.path, MAX_FRAME_BYTES).await?; + let stored = put_file_pieces(net, store, &file.path, limit).await?; metas.push(FrameMeta { frame: f.frame, format: f.info.format.name().into(), @@ -1169,7 +1205,7 @@ async fn run_render_job( if bytes.len() > MAX_RESULT_BYTES { bail!("render result is over the {MAX_RESULT_BYTES}-byte limit"); } - crate::pieces::publish_result_bytes(net, store, &bytes) + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes) .await .context("publish render result") } @@ -1182,6 +1218,7 @@ pub async fn fetch_frames( net: &Arc, worker: &str, frames: &[FrameMeta], + key: &c0mpute_net::topics::JobKey, budget: Duration, ) -> Result)>> { let started = Instant::now(); @@ -1207,8 +1244,15 @@ pub async fn fetch_frames( bail!("frame {} is bigger than its result says", meta.frame); } } - verify_frame(meta, &bytes)?; - out.push((meta.clone(), bytes)); + // `size` and `hash` name the sealed frame; the image is checked once + // it is opened with the job's result key. + if bytes.len() as u64 != meta.size || Hash::of(&bytes).to_hex() != meta.hash { + bail!("frame {} does not match its size or hash", meta.frame); + } + let plain = crate::job_e2e::open_blob(key, &bytes) + .with_context(|| format!("frame {} does not open with this job's key", meta.frame))?; + check_frame_image(meta, &plain)?; + out.push((meta.clone(), plain)); } Ok(out) } @@ -1274,7 +1318,10 @@ pub async fn render_frames( bail!("no frames to render"); } tokio::fs::create_dir_all(&job.out_dir).await?; - let input = put_file_pieces(&net, store, &job.blend, MAX_INPUT_BYTES).await?; + // Sealed under a fresh key that only the winning workers learn. + let (input, input_key) = + crate::pieces::put_file_pieces_sealed(&net, store, &job.blend, MAX_INPUT_BYTES, PIECE_BYTES).await?; + let input_key = crate::job_e2e::key_to_hex(&input_key); progress(&format!( "published {} ({} bytes, {} pieces, blake3 {})", job.blend.display(), @@ -1296,6 +1343,7 @@ pub async fn render_frames( "input_size": input.size, "input_pieces": input.pieces, "input_provider_peer_id": provider, + "input_key": input_key, "frame_start": start, "frame_end": end, "engine": job.engine.name(), @@ -1417,10 +1465,10 @@ async fn collect_batch( .output_hash .as_deref() .context("worker returned no output hash")?; + let _ = hash; let bytes = crate::gateway_util::fetch_result( net, - hash, - worker, + outcome, MAX_RESULT_BYTES, Duration::from_secs(30), ) @@ -1430,7 +1478,7 @@ async fn collect_batch( let metas = result_frames(parsed, &result)?; let budget = Duration::from_secs(120 + 30 * metas.iter().map(|m| m.pieces.len() as u64).sum::()); - let frames = fetch_frames(net, worker, &metas, budget).await?; + let frames = fetch_frames(net, worker, &metas, &outcome.result_key, budget).await?; // Every frame of one render has the same size. { let mut known = dims.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/node/crates/c0mpute-core/src/routing.rs b/node/crates/c0mpute-core/src/routing.rs index 38c64aab..b2d7d5aa 100644 --- a/node/crates/c0mpute-core/src/routing.rs +++ b/node/crates/c0mpute-core/src/routing.rs @@ -135,6 +135,7 @@ mod tests { expires_at_ms: None, bench_score: bench, load, + e2e: None, } } diff --git a/node/crates/c0mpute-core/src/runner.rs b/node/crates/c0mpute-core/src/runner.rs index fbdcd841..bd54d3f8 100644 --- a/node/crates/c0mpute-core/src/runner.rs +++ b/node/crates/c0mpute-core/src/runner.rs @@ -147,6 +147,10 @@ pub struct SttJobInline { /// Peer that owns the input audio, supplied by the buyer. #[serde(default)] pub audio_provider_peer_id: Option, + /// Hex key the stored audio is sealed under (DIP-0032); rides only inside + /// the sealed spec. Absent: the stored bytes are plaintext. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub audio_key: Option, #[serde(default = "default_stt_model")] pub model: String, #[serde(default)] @@ -271,7 +275,7 @@ async fn run_tts_job( { bail!("TTS output is not a valid WAV under the 20 MiB result limit"); } - crate::pieces::publish_result_bytes(net, store, &bytes).await + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes).await } /// Largest audio input a worker accepts. @@ -303,12 +307,13 @@ async fn run_stt_job( if !crate::speech::stt_ready() { bail!("faster-whisper is not installed; run `c0mpute stt setup`"); } + let key = crate::pieces::input_key(inline.audio_key.as_deref())?; let input = crate::pieces::InputRef::parse( "audio", &inline.audio_hash, inline.audio_size, inline.audio_pieces.as_deref(), - MAX_STT_AUDIO_BYTES, + crate::pieces::input_limit(MAX_STT_AUDIO_BYTES, key.is_some()), ) .map_err(anyhow::Error::msg)?; let provider = inline @@ -318,7 +323,7 @@ async fn run_stt_job( .transpose() .context("invalid audio provider peer id")?; let audio = input - .fetch(net, provider, MAX_STT_AUDIO_BYTES) + .fetch_opened(net, provider, MAX_STT_AUDIO_BYTES, key.as_ref()) .await .context("fetch the audio input (maximum 100 MiB)")?; let job_dir = safe_job_dir(cache_root, &job.accept.job_id)?; @@ -332,7 +337,7 @@ async fn run_stt_job( if bytes.is_empty() || bytes.len() > 10 * 1024 * 1024 { bail!("STT result is empty or exceeds the 10 MiB result limit"); } - crate::pieces::publish_result_bytes(net, store, &bytes).await + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes).await } pub(crate) fn safe_job_dir(cache_root: &std::path::Path, job_id: &str) -> Result { @@ -441,7 +446,7 @@ async fn run_transcode_job( let _ = fs::remove_file(&input_path).await; // Streamed from disk as pieces (a manifest is what the receipt names); // the output is never in memory whole. - crate::pieces::publish_result_file(net, store, &output_path, MAX_OUTPUT_BYTES) + crate::pieces::publish_result_file(net, store, &job.accept, &output_path, MAX_OUTPUT_BYTES) .await .context("publish transcode output") } diff --git a/node/crates/c0mpute-core/src/speech_gateway.rs b/node/crates/c0mpute-core/src/speech_gateway.rs index 85d3f464..48f6f096 100644 --- a/node/crates/c0mpute-core/src/speech_gateway.rs +++ b/node/crates/c0mpute-core/src/speech_gateway.rs @@ -13,7 +13,7 @@ use axum::http::{HeaderMap, HeaderValue, StatusCode, header}; use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post}; use axum::{Json, Router}; -use c0mpute_net::{Libp2pNetwork, Network}; +use c0mpute_net::Libp2pNetwork; use c0mpute_store::ChunkStore; use serde::{Deserialize, Serialize}; use tokio::sync::Semaphore; @@ -508,7 +508,8 @@ async fn transcribe( ) })?; // One network message carries ~5 MiB at most: store the audio in pieces. - let audio = crate::pieces::put_bytes_pieces( + // Sealed under a fresh key that only the winning worker learns. + let (audio, audio_key) = crate::pieces::put_bytes_pieces_sealed( &state.net, &state.store, &body, @@ -524,7 +525,7 @@ async fn transcribe( }); spec.as_object_mut() .expect("spec is an object") - .extend(audio.spec_fields("audio")); + .extend(audio.sealed_spec_fields("audio", &audio_key)); let (outcome, bytes) = run_speech_job( &state, "stt.transcribe", @@ -741,8 +742,8 @@ async fn run_checked_job( StatusCode::BAD_GATEWAY, format!("c0mpute worker failed the {workload} job"), )) - } else if let Some(output_hash) = outcome.receipt.output_hash.as_deref() { - match fetch_result(state, output_hash, &worker, max_bytes).await { + } else if outcome.receipt.output_hash.is_some() { + match fetch_result(state, &outcome, max_bytes).await { Ok(bytes) if valid(&bytes) => Ok(bytes), Ok(_) => Err(ApiError( StatusCode::BAD_GATEWAY, @@ -834,15 +835,13 @@ fn result_fetch_budget(max_bytes: usize) -> Duration { async fn fetch_result( state: &GatewayState, - hash: &str, - provider_peer_id: &str, + outcome: &crate::AuctionOutcome, max_bytes: usize, ) -> ApiResult> { use crate::gateway_util::FetchError; match crate::gateway_util::fetch_result( &state.net, - hash, - provider_peer_id, + outcome, max_bytes, result_fetch_budget(max_bytes), ) @@ -1088,7 +1087,7 @@ async fn generate_video( "image must be 1 byte to 32 MiB".into(), )); } - let image = crate::pieces::put_bytes_pieces( + let (image, image_key) = crate::pieces::put_bytes_pieces_sealed( &state.net, &state.store, &bytes, @@ -1096,6 +1095,7 @@ async fn generate_video( ) .await .map_err(gateway_failure)?; + job.image_key = Some(crate::job_e2e::key_to_hex(&image_key)); job.image_hash = Some(image.hash.clone()); if image.pieces.len() > 1 { job.image_size = Some(image.size); @@ -1557,7 +1557,7 @@ async fn ocr_extract( ApiError(StatusCode::SERVICE_UNAVAILABLE, "OCR queue is unavailable".into()) })?; // One network message carries ~5 MiB at most: store the document in pieces. - let input = crate::pieces::put_bytes_pieces( + let (input, input_key) = crate::pieces::put_bytes_pieces_sealed( &state.net, &state.store, &body, @@ -1573,7 +1573,7 @@ async fn ocr_extract( }); spec.as_object_mut() .expect("spec is an object") - .extend(input.spec_fields("input")); + .extend(input.sealed_spec_fields("input", &input_key)); let (outcome, bytes) = run_speech_job( &state, crate::ocr::WORKLOAD, @@ -1670,7 +1670,7 @@ async fn captcha_solve( let _slot = state.stt_slots.clone().acquire_owned().await.map_err(|_| { ApiError(StatusCode::SERVICE_UNAVAILABLE, "speech queue is unavailable".into()) })?; - let pieces = crate::pieces::put_bytes_pieces( + let (pieces, audio_key) = crate::pieces::put_bytes_pieces_sealed( &state.net, &state.store, &audio, @@ -1686,7 +1686,7 @@ async fn captcha_solve( }); spec.as_object_mut() .expect("spec is an object") - .extend(pieces.spec_fields("audio")); + .extend(pieces.sealed_spec_fields("audio", &audio_key)); let (outcome, bytes) = run_speech_job( &state, "stt.transcribe", @@ -1746,17 +1746,10 @@ async fn captcha_solve( let _slot = state.captcha_slots.clone().acquire_owned().await.map_err(|_| { ApiError(StatusCode::SERVICE_UNAVAILABLE, "captcha queue is unavailable".into()) })?; - let spec_bytes = - serde_json::to_vec(&spec).map_err(|error| gateway_failure(error.into()))?; - // Up to two 256 KiB images: often too big for the gossip offer, so - // the worker fetches the spec from this node by hash. - let offer_spec = if spec_bytes.len() <= crate::workflow::MAX_INLINE_SPEC_BYTES { - spec - } else { - let hash = state.store.put(&spec_bytes).await.map_err(gateway_failure)?; - state.net.announce(&hash).await.map_err(gateway_failure)?; - crate::workflow::spec_ref(&hash, spec_bytes.len()) - }; + // The spec (up to two 256 KiB images) never rides the gossip + // offer: it is sealed to the winner and fetched point to point + // (DIP-0032), so it is never stored here as a plaintext chunk. + let offer_spec = spec; let expected = parsed.kind.clone(); let (outcome, bytes) = run_checked_job( &state, @@ -1858,16 +1851,9 @@ async fn embeddings( let _slot = state.embed_slots.clone().acquire_owned().await.map_err(|_| { ApiError(StatusCode::SERVICE_UNAVAILABLE, "embeddings queue is unavailable".into()) })?; - // Up to 1 MiB of text: too big for the gossip offer, so the worker - // fetches it from this node by hash. - let spec_bytes = serde_json::to_vec(&spec).map_err(|error| gateway_failure(error.into()))?; - let offer_spec = if spec_bytes.len() <= crate::workflow::MAX_INLINE_SPEC_BYTES { - spec - } else { - let hash = state.store.put(&spec_bytes).await.map_err(gateway_failure)?; - state.net.announce(&hash).await.map_err(gateway_failure)?; - crate::workflow::spec_ref(&hash, spec_bytes.len()) - }; + // Up to 1 MiB of text, sealed to the winner and fetched point to point + // (DIP-0032): never on gossip, never stored here as a plaintext chunk. + let offer_spec = spec; let (outcome, bytes) = run_checked_job( &state, crate::embed::WORKLOAD, diff --git a/node/crates/c0mpute-core/src/status_aggregator.rs b/node/crates/c0mpute-core/src/status_aggregator.rs index a08108fc..5dd2af24 100644 --- a/node/crates/c0mpute-core/src/status_aggregator.rs +++ b/node/crates/c0mpute-core/src/status_aggregator.rs @@ -954,6 +954,8 @@ mod tests { published_at_ms: now - 5_000, external_ref: None, buyer_addrs: Vec::new(), + e2e: None, + result_key: None, }) .await; assert_eq!(jobs.in_flight.read().await.len(), 1); @@ -992,6 +994,8 @@ mod tests { published_at_ms: now, external_ref: None, buyer_addrs: Vec::new(), + e2e: None, + result_key: None, }) .await; jobs.on_receipt(JobReceipt { diff --git a/node/crates/c0mpute-core/src/supervisor.rs b/node/crates/c0mpute-core/src/supervisor.rs index 00b61e37..9a7eb37b 100644 --- a/node/crates/c0mpute-core/src/supervisor.rs +++ b/node/crates/c0mpute-core/src/supervisor.rs @@ -483,11 +483,18 @@ impl Supervisor { ); let gate = config.video.clone(); let filter: dispatch::OfferFilter = std::sync::Arc::new(move |offer| { - let fits = offer - .spec_inline - .as_ref() + // An e2e offer's prompt stays sealed: a stand-in lets + // the size checks run; the opened spec is checked again. + let spec = match crate::job_e2e::bid_spec(offer) { + Some(crate::job_e2e::BidSpec::Public(v)) => { + Some(crate::job_e2e::with_placeholders(v, &[("prompt", serde_json::json!("x"))])) + } + Some(crate::job_e2e::BidSpec::Full(v)) => Some(v.clone()), + None => None, + }; + let fits = spec .and_then(|spec| { - serde_json::from_value::(spec.clone()).ok() + serde_json::from_value::(spec).ok() }) .is_some_and(|job| { crate::video_cpu::fits(&job, crate::video::WORKLOAD_TXT2VID).is_ok() diff --git a/node/crates/c0mpute-core/src/temporal/worker.rs b/node/crates/c0mpute-core/src/temporal/worker.rs index f8d6ee3c..18ccfdc5 100644 --- a/node/crates/c0mpute-core/src/temporal/worker.rs +++ b/node/crates/c0mpute-core/src/temporal/worker.rs @@ -431,21 +431,35 @@ pub fn train_offer_filter(busy: Arc) -> crate::di if busy.load(std::sync::atomic::Ordering::Relaxed) { return false; } - offer - .spec_inline - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|s| s.validate().is_ok()) + match crate::job_e2e::bid_spec(offer) { + // The dataset stays sealed: judge the job's schema, family and + // privacy approval; the opened spec is validated in full. + Some(crate::job_e2e::BidSpec::Public(v)) => { + let j = &v["train_job"]; + j["schema_version"].as_str() == Some(super::SCHEMA_VERSION) + && j["kind"].as_str() == Some("train_job") + && super::FAMILIES.contains(&j["model_type"].as_str().unwrap_or("")) + && j["privacy"]["mode"].as_str() == Some("trusted_worker") + && j["privacy"]["upload_approved"].as_bool() == Some(true) + } + Some(crate::job_e2e::BidSpec::Full(v)) => { + serde_json::from_value::(v.clone()).ok().is_some_and(|s| s.validate().is_ok()) + } + None => false, + } }) } /// Bid only on session offers for models this node serves. pub fn infer_offer_filter(models: Vec) -> crate::dispatch::OfferFilter { - Arc::new(move |offer| { - offer - .spec_inline - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|s| s.validate().is_ok() && models.contains(&s.model_hash)) + Arc::new(move |offer| match crate::job_e2e::bid_spec(offer) { + Some(crate::job_e2e::BidSpec::Public(v)) => { + v["model_hash"].as_str().is_some_and(|m| models.iter().any(|x| x == m)) + && v["ttl_s"].as_u64().is_some_and(|t| (1..=86_400).contains(&t)) + } + Some(crate::job_e2e::BidSpec::Full(v)) => serde_json::from_value::(v.clone()) + .ok() + .is_some_and(|s| s.validate().is_ok() && models.contains(&s.model_hash)), + None => false, }) } diff --git a/node/crates/c0mpute-core/src/text.rs b/node/crates/c0mpute-core/src/text.rs index 2746fb93..4459e6ee 100644 --- a/node/crates/c0mpute-core/src/text.rs +++ b/node/crates/c0mpute-core/src/text.rs @@ -134,6 +134,10 @@ pub struct TextTrainSpec { pub corpus_size: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub corpus_pieces: Option>, + /// Hex key the stored corpus is sealed under (DIP-0032); rides only inside + /// the sealed spec. Absent: the stored bytes are plaintext. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub corpus_key: Option, /// Return the adapter as pieces behind an [`AdapterManifest`]. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub adapter_pieces: bool, @@ -205,7 +209,7 @@ impl TextTrainSpec { pub fn corpus_ref(&self) -> Option> { let hash = self.corpus_hash.as_deref()?; Some( - crate::pieces::InputRef::parse("corpus", hash, self.corpus_size, self.corpus_pieces.as_deref(), MAX_DATASET as usize) + crate::pieces::InputRef::parse("corpus", hash, self.corpus_size, self.corpus_pieces.as_deref(), crate::pieces::input_limit(MAX_DATASET as usize, self.corpus_key.is_some())) .map_err(anyhow::Error::msg), ) } @@ -302,7 +306,8 @@ async fn download(url: &str, dst: &Path, want: &str) -> Result<()> { pub async fn fetch_corpus(net: &Arc, buyer: Option, spec: &TextTrainSpec, dst: &Path) -> Result<()> { match spec.corpus_ref() { Some(r) => { - r?.fetch_to_file(net, buyer, MAX_DATASET as usize, dst).await.context("fetch corpus pieces")?; + let key = crate::pieces::input_key(spec.corpus_key.as_deref())?; + r?.fetch_to_file_opened(net, buyer, MAX_DATASET as usize, key.as_ref(), dst).await.context("fetch corpus pieces")?; let got = crate::temporal::worker::file_sha256(dst)?; if got != spec.dataset_hash { let _ = tokio::fs::remove_file(dst).await; @@ -457,7 +462,10 @@ async fn upload(base: &str, dir: &Path) -> Result<()> { /// Store the adapter files as pieces, then the [`AdapterManifest`] naming /// them; returns the manifest's content hash (what the receipt carries). The /// pieces are dropped from the store after [`ADAPTER_PIECES_TTL`]. -pub async fn store_adapter(net: &Arc, store: &ChunkStore, dir: &Path, adapter_sha256: &str) -> Result { +/// +/// With `key` (DIP-0032) every file and the manifest are sealed under it +/// first: what is stored, announced and named is ciphertext. +pub async fn store_adapter(net: &Arc, store: &ChunkStore, dir: &Path, adapter_sha256: &str, key: Option<&c0mpute_net::topics::JobKey>) -> Result { let mut files = std::collections::BTreeMap::new(); let mut held = Vec::new(); for f in ARTIFACTS { @@ -465,14 +473,25 @@ pub async fn store_adapter(net: &Arc, store: &ChunkStore, dir: &P if !p.exists() { continue; } - let stored = crate::pieces::put_file_pieces(net, store, &p, MAX_ADAPTER_FILE_BYTES as u64, crate::pieces::INPUT_PIECE_BYTES).await.with_context(|| format!("store {f}"))?; + let (src, limit, _sealed) = match key { + Some(k) => { + let sealed = std::env::temp_dir().join(format!("c0mpute-adapter-{}.sealed", uuid::Uuid::new_v4())); + crate::job_e2e::seal_file(k, &p, &sealed).await?; + (sealed.clone(), crate::job_e2e::sealed_len(MAX_ADAPTER_FILE_BYTES as u64), Some(RemoveOnDrop(sealed))) + } + None => (p.clone(), MAX_ADAPTER_FILE_BYTES as u64, None), + }; + let stored = crate::pieces::put_file_pieces(net, store, &src, limit, crate::pieces::INPUT_PIECE_BYTES).await.with_context(|| format!("store {f}"))?; held.extend(stored.pieces.iter().cloned()); files.insert(f.to_string(), ManifestFile { hash: stored.hash, size: stored.size, pieces: stored.pieces }); } if !files.contains_key("adapter_model.safetensors") { bail!("no adapter_model.safetensors to store"); } - let manifest = serde_json::to_vec(&AdapterManifest { v: 1, adapter_sha256: adapter_sha256.to_string(), files })?; + let mut manifest = serde_json::to_vec(&AdapterManifest { v: 1, adapter_sha256: adapter_sha256.to_string(), files })?; + if let Some(k) = key { + manifest = crate::job_e2e::seal_blob(k, &manifest)?; + } let hash = store.put(&manifest).await.context("store adapter manifest")?; net.announce(&hash).await.context("announce adapter manifest")?; held.push(hash.to_hex()); @@ -488,6 +507,24 @@ pub async fn store_adapter(net: &Arc, store: &ChunkStore, dir: &P Ok(hash.to_hex()) } +/// Removes a temporary file when dropped. +struct RemoveOnDrop(PathBuf); + +impl Drop for RemoveOnDrop { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// Removes a job directory when dropped, on every exit path. +struct RemoveDirOnDrop(PathBuf); + +impl Drop for RemoveDirOnDrop { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root: PathBuf, infernet: PathBuf, busy: Arc, slots: Arc, in_flight: InFlight) -> mpsc::Sender { let (tx, mut rx) = mpsc::channel::(8); tokio::spawn(async move { @@ -504,7 +541,9 @@ pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root let (dir, hash) = run_train(&infernet, &job_dir, &spec, &net, buyer).await?; if spec.adapter_pieces { // Receipt -> manifest -> pieces, all over libp2p. - return store_adapter(&net, &store, &dir, &hash).await; + // Sealed to the buyer for e2e jobs. + let key = crate::job_e2e::result_key(&accept)?; + return store_adapter(&net, &store, &dir, &hash, key).await; } upload(&spec.artifact_upload_url, &dir).await.context("upload adapter")?; Ok(hash) @@ -532,11 +571,19 @@ pub fn spawn_train_runner(net: Arc, store: ChunkStore, cache_root pub fn train_offer_filter(busy: Arc) -> crate::dispatch::OfferFilter { Arc::new(move |offer| { !busy.load(Ordering::Relaxed) - && offer - .spec_inline - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|s| s.validate().is_ok()) + && match crate::job_e2e::bid_spec(offer) { + // The corpus and URLs stay sealed: judge the kind and base + // model; the opened spec is validated in full before it runs. + Some(crate::job_e2e::BidSpec::Public(v)) => { + let kind = v["job"]["kind"].as_str().unwrap_or(""); + let base = v["job"]["base_model"].as_str().unwrap_or(""); + KINDS.contains(&kind) && valid_repo_id(base) + } + Some(crate::job_e2e::BidSpec::Full(v)) => { + serde_json::from_value::(v.clone()).ok().is_some_and(|s| s.validate().is_ok()) + } + None => false, + } }) } @@ -583,6 +630,11 @@ pub struct TextInferSpec { /// Return `answer.json` as a chunk (receipt `output_hash` = its content hash). #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub answer_chunk: bool, + /// Hex key the adapter pieces and manifest are sealed under (DIP-0032). + /// With it the worker keeps the adapter only for this job, never in its + /// cache. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub adapter_key: Option, } fn valid_repo_id(base: &str) -> bool { @@ -620,6 +672,9 @@ impl TextInferSpec { if self.adapter_manifest.as_deref().is_some_and(|h| c0mpute_proto::Hash::from_hex(h).is_err()) { bail!("adapter_manifest must be a hex content hash"); } + if self.adapter_key.as_deref().is_some_and(|k| crate::job_e2e::key_from_hex(k).is_err()) { + bail!("adapter_key must be 32 bytes of hex"); + } if self.adapter_manifest.is_none() && self.adapter_url.is_empty() { bail!("the spec names no adapter: adapter_manifest or adapter_url"); } @@ -679,13 +734,22 @@ async fn fetch_adapter(url: &str, want: &str, dir: &Path) -> Result<()> { /// `buyer` is the peer adapter pieces are fetched from first. pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: &TextInferSpec, net: &Arc, buyer: Option) -> Result> { spec.validate()?; - let adapter = cache_root.join("text").join("adapters").join(spec.adapter_hash.trim_start_matches("sha256:")); + // A sealed adapter (DIP-0032) is the buyer's private data: it lives only + // in this job's directory, removed when the job ends, never in the cache. + let input_key = spec.adapter_key.as_deref().map(crate::job_e2e::key_from_hex).transpose()?; + let (adapter, _private_adapter) = match &input_key { + Some(_) => { + let d = cache_root.join("text").join("infer-adapters").join(job_id); + (d.clone(), Some(RemoveDirOnDrop(d))) + } + None => (cache_root.join("text").join("adapters").join(spec.adapter_hash.trim_start_matches("sha256:")), None), + }; let cached = crate::temporal::worker::file_sha256(&adapter.join("adapter_model.safetensors")).ok(); if cached.as_deref() != Some(spec.adapter_hash.as_str()) { match &spec.adapter_manifest { Some(m) => { let _ = tokio::fs::remove_dir_all(&adapter).await; - let got = fetch_adapter_pieces(net, buyer, m, &adapter).await.context("fetch adapter pieces")?; + let got = fetch_adapter_pieces(net, buyer, m, &adapter, input_key.as_ref()).await.context("fetch adapter pieces")?; if got != spec.adapter_hash { let _ = tokio::fs::remove_dir_all(&adapter).await; bail!("adapter {got} differs from the offer's {}", spec.adapter_hash); @@ -696,6 +760,8 @@ pub async fn run_infer(infernet: &Path, cache_root: &Path, job_id: &str, spec: & } let dir = cache_root.join("text").join("infer").join(job_id); tokio::fs::create_dir_all(&dir).await?; + // The request holds the prompt: removed on every exit, timeouts included. + let _job_dir = RemoveDirOnDrop(dir.clone()); let req = dir.join("request.json"); tokio::fs::write(&req, serde_json::to_vec(&json!({"adapter_dir": adapter, "base_model": spec.base_model, "messages": spec.messages, "max_new_tokens": spec.max_new_tokens}))?).await?; let mut command = tokio::process::Command::new(crate::live_bin(infernet)); @@ -725,6 +791,8 @@ pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root let buyer = accept.buyer_peer_id.parse::().ok(); let answer = run_infer(&infernet, &cache_root, &accept.job_id, &spec, &net, buyer).await?; if spec.answer_chunk { + // Sealed to the buyer for e2e jobs (one chunk either way). + let answer = crate::job_e2e::seal_result(&accept, &answer)?.unwrap_or(answer); let hash = store.put(&answer).await.context("store answer")?; net.announce(&hash).await.context("announce answer")?; return Ok(hash.to_hex()); @@ -752,11 +820,16 @@ pub fn spawn_infer_runner(net: Arc, store: ChunkStore, cache_root pub fn infer_offer_filter(busy: Arc) -> crate::dispatch::OfferFilter { Arc::new(move |offer| { !busy.load(Ordering::Relaxed) - && offer - .spec_inline - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) - .is_some_and(|s| s.validate().is_ok()) + && match crate::job_e2e::bid_spec(offer) { + Some(crate::job_e2e::BidSpec::Public(v)) => { + valid_repo_id(v["base_model"].as_str().unwrap_or("")) + && v["max_new_tokens"].as_u64().is_some_and(|n| (1..=1024).contains(&n)) + } + Some(crate::job_e2e::BidSpec::Full(v)) => { + serde_json::from_value::(v.clone()).ok().is_some_and(|s| s.validate().is_ok()) + } + None => false, + } }) } @@ -1019,7 +1092,8 @@ pub async fn train(net: Arc, store: &ChunkStore, req: &TrainReque if replicas > 1 && req.legacy_http { bail!("--replicas needs the peer-to-peer transport; drop --legacy-http"); } - let stored = crate::pieces::put_bytes_pieces(&net, store, &bytes, crate::pieces::INPUT_PIECE_BYTES).await.context("store the corpus as pieces")?; + // Sealed under a fresh key that only the winning worker learns. + let (stored, corpus_key) = crate::pieces::put_bytes_pieces_sealed(&net, store, &bytes, crate::pieces::INPUT_PIECE_BYTES).await.context("store the corpus as pieces")?; let run_id = uuid::Uuid::new_v4().to_string(); let mut spec = TextTrainSpec { job, @@ -1030,6 +1104,7 @@ pub async fn train(net: Arc, store: &ChunkStore, req: &TrainReque corpus_hash: Some(stored.hash.clone()), corpus_size: (stored.pieces.len() > 1).then_some(stored.size), corpus_pieces: (stored.pieces.len() > 1).then(|| stored.pieces.clone()), + corpus_key: Some(crate::job_e2e::key_to_hex(&corpus_key)), adapter_pieces: true, run_id: Some(run_id.clone()), }; @@ -1146,7 +1221,7 @@ pub async fn train(net: Arc, store: &ChunkStore, req: &TrainReque (got, "http") } else { let worker = o.receipt.worker_peer_id.parse::().ok(); - (fetch_adapter_pieces(&net, worker, &claimed, &req.output).await?, "pieces") + (fetch_adapter_pieces(&net, worker, &claimed, &req.output, Some(&o.result_key)).await?, "pieces") }; Ok(json!({ "adapter_dir": req.output, "adapter_hash": got, "base_model": preview.base_model, "dataset_hash": preview.dataset_hash, @@ -1349,7 +1424,7 @@ async fn train_replicated(net: &Arc, req: &TrainRequest, spec: &T let dir = scratch.join(k.to_string()); let claimed = o.receipt.output_hash.clone().unwrap_or_default(); let fetched = async { - let hash = fetch_adapter_pieces(net, worker.parse::().ok(), &claimed, &dir).await?; + let hash = fetch_adapter_pieces(net, worker.parse::().ok(), &claimed, &dir, Some(&o.result_key)).await?; let r = Replica::from_dir(&worker, &dir, evals.get(&k).map(|(_, p)| *p))?; Ok::<_, anyhow::Error>((hash, r)) } @@ -1409,9 +1484,17 @@ async fn train_replicated(net: &Arc, req: &TrainRequest, spec: &T /// Fetch the [`AdapterManifest`] a receipt names, then every file it lists, /// into `out`; keep them only if adapter_model.safetensors hashes to the /// manifest's `adapter_sha256`. Returns that hash. -pub async fn fetch_adapter_pieces(net: &Arc, worker: Option, manifest_hash: &str, out: &Path) -> Result { - let mref = crate::pieces::InputRef::parse("adapter_manifest", manifest_hash, None, None, 1 << 20).map_err(anyhow::Error::msg)?; - let manifest: AdapterManifest = serde_json::from_slice(&mref.fetch(net, worker, 1 << 20).await.context("fetch the adapter manifest")?).context("adapter manifest is not JSON")?; +/// +/// With `key` the manifest and files are sealed (see [`store_adapter`]): +/// each is opened after its hash checks. +pub async fn fetch_adapter_pieces(net: &Arc, worker: Option, manifest_hash: &str, out: &Path, key: Option<&c0mpute_net::topics::JobKey>) -> Result { + let mmax = if key.is_some() { crate::job_e2e::sealed_len(1 << 20) as usize } else { 1 << 20 }; + let mref = crate::pieces::InputRef::parse("adapter_manifest", manifest_hash, None, None, mmax).map_err(anyhow::Error::msg)?; + let mut mbytes = mref.fetch(net, worker, mmax).await.context("fetch the adapter manifest")?; + if let Some(k) = key { + mbytes = crate::job_e2e::open_blob(k, &mbytes).context("adapter manifest does not open with the job key")?; + } + let manifest: AdapterManifest = serde_json::from_slice(&mbytes).context("adapter manifest is not JSON")?; if manifest.v != 1 || !crate::temporal::is_sha256_ref(&manifest.adapter_sha256) { bail!("adapter manifest has an unknown version or a bad adapter_sha256"); } @@ -1423,8 +1506,18 @@ pub async fn fetch_adapter_pieces(net: &Arc, worker: Option = async { for (name, f) in &manifest.files { let (size, pieces) = if f.pieces.len() > 1 { (Some(f.size), Some(f.pieces.as_slice())) } else { (None, None) }; - let r = crate::pieces::InputRef::parse("adapter", &f.hash, size, pieces, MAX_ADAPTER_FILE_BYTES).map_err(anyhow::Error::msg)?; - r.fetch_to_file(net, worker, MAX_ADAPTER_FILE_BYTES, &tmp.join(name)).await.with_context(|| format!("fetch {name}"))?; + let fmax = if key.is_some() { crate::job_e2e::sealed_len(MAX_ADAPTER_FILE_BYTES as u64) as usize } else { MAX_ADAPTER_FILE_BYTES }; + let r = crate::pieces::InputRef::parse("adapter", &f.hash, size, pieces, fmax).map_err(anyhow::Error::msg)?; + match key { + Some(k) => { + let sealed = tmp.join(format!("{name}.sealed")); + r.fetch_to_file(net, worker, fmax, &sealed).await.with_context(|| format!("fetch {name}"))?; + let opened = crate::job_e2e::open_file(k, &sealed, &tmp.join(name)).await; + let _ = tokio::fs::remove_file(&sealed).await; + opened.with_context(|| format!("{name} does not open with the job key"))?; + } + None => r.fetch_to_file(net, worker, fmax, &tmp.join(name)).await.with_context(|| format!("fetch {name}"))?, + } } let got = crate::temporal::worker::file_sha256(&tmp.join("adapter_model.safetensors"))?; if got != manifest.adapter_sha256 { @@ -1524,7 +1617,10 @@ pub async fn chat(net: Arc, store: &ChunkStore, req: &ChatRequest messages.push(json!({"role": "system", "content": system})); } messages.push(json!({"role": "user", "content": chat_prompt(&req.question, &sources)})); - let adapter_manifest = store_adapter(&net, store, &req.adapter_dir, &adapter_hash).await.context("store the adapter as pieces")?; + // The adapter is the buyer's model: sealed under a fresh key that only + // the winning worker learns (inside the sealed spec). + let adapter_key = crate::job_e2e::random_key(); + let adapter_manifest = store_adapter(&net, store, &req.adapter_dir, &adapter_hash, Some(&adapter_key)).await.context("store the adapter as pieces")?; let mut spec = TextInferSpec { adapter_hash: adapter_hash.clone(), base_model: base_model.clone(), @@ -1534,6 +1630,7 @@ pub async fn chat(net: Arc, store: &ChunkStore, req: &ChatRequest answer_upload_url: String::new(), adapter_manifest: Some(adapter_manifest), answer_chunk: true, + adapter_key: Some(crate::job_e2e::key_to_hex(&adapter_key)), }; let server = if req.legacy_http { let s = TransferServer::start_with(req.bind, req.public_url.clone(), bundle, &ANSWER_ARTIFACTS, None).await?; @@ -1571,8 +1668,10 @@ pub async fn chat(net: Arc, store: &ChunkStore, req: &ChatRequest answer } else { // The fetch checks the bytes against the receipt's content hash. - let r = crate::pieces::InputRef::parse("answer", &claimed, None, None, 1 << 20).map_err(anyhow::Error::msg)?; - r.fetch(&net, o.receipt.worker_peer_id.parse::().ok(), 1 << 20).await.context("fetch the answer")? + let max = crate::job_e2e::sealed_len(1 << 20) as usize; + let r = crate::pieces::InputRef::parse("answer", &claimed, None, None, max).map_err(anyhow::Error::msg)?; + let sealed = r.fetch(&net, o.receipt.worker_peer_id.parse::().ok(), max).await.context("fetch the answer")?; + o.open_result(&sealed).context("the answer does not open with this job's key")? }; let ans: Value = serde_json::from_slice(&answer)?; if ans["adapter"].as_str().is_some_and(|a| a != adapter_hash) { @@ -1617,6 +1716,7 @@ mod tests { fn infer_spec() -> TextInferSpec { TextInferSpec { + adapter_key: None, adapter_hash: format!("sha256:{}", "b".repeat(64)), base_model: "HuggingFaceTB/SmolLM2-135M-Instruct".into(), messages: vec![json!({"role": "user", "content": "hi"})], @@ -1699,6 +1799,7 @@ mod tests { fn spec() -> TextTrainSpec { TextTrainSpec { + corpus_key: None, job: json!({"kind": "lora", "base_model": "HuggingFaceTB/SmolLM2-135M-Instruct"}), dataset_url: "http://127.0.0.1:1/t/x/dataset.json".into(), dataset_hash: format!("sha256:{}", "a".repeat(64)), @@ -1714,6 +1815,7 @@ mod tests { fn p2p_spec() -> TextTrainSpec { TextTrainSpec { + corpus_key: None, dataset_url: String::new(), artifact_upload_url: String::new(), events_url: None, diff --git a/node/crates/c0mpute-core/src/train.rs b/node/crates/c0mpute-core/src/train.rs index 1f725a40..b1de70cf 100644 --- a/node/crates/c0mpute-core/src/train.rs +++ b/node/crates/c0mpute-core/src/train.rs @@ -350,6 +350,7 @@ mod tests { assert_eq!(job["system_prompt"], "from data"); // The same spec validation the worker runs. let spec = crate::text::TextTrainSpec { + corpus_key: None, job: job.clone(), dataset_url: String::new(), dataset_hash: format!("sha256:{}", "a".repeat(64)), diff --git a/node/crates/c0mpute-core/src/video.rs b/node/crates/c0mpute-core/src/video.rs index 049ccb38..6ee1bb00 100644 --- a/node/crates/c0mpute-core/src/video.rs +++ b/node/crates/c0mpute-core/src/video.rs @@ -249,6 +249,10 @@ pub struct VideoJobInline { /// Peer holding `image_hash`, when the buyer knows it. #[serde(default)] pub image_provider_peer_id: Option, + /// Hex key the stored image is sealed under (DIP-0032); rides only inside + /// the sealed spec. Absent: the stored bytes are plaintext. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_key: Option, #[serde(default = "default_image_format")] pub image_format: String, #[serde(default = "default_frames")] @@ -295,7 +299,7 @@ impl VideoJobInline { hash, self.image_size, self.image_pieces.as_deref(), - MAX_IMAGE_BYTES, + crate::pieces::input_limit(MAX_IMAGE_BYTES, self.image_key.is_some()), ) }) } @@ -684,8 +688,9 @@ async fn run_video_job( .map(str::parse) .transpose() .context("invalid image provider peer id")?; + let key = crate::pieces::input_key(inline.image_key.as_deref())?; let bytes = input - .fetch(net, provider, MAX_IMAGE_BYTES) + .fetch_opened(net, provider, MAX_IMAGE_BYTES, key.as_ref()) .await .context("fetch the conditioning image (maximum 32 MiB)")?; let name = format!("input.{}", inline.image_format); @@ -737,7 +742,7 @@ async fn run_video_job( bail!("video result is not an MP4"); } } - crate::pieces::publish_result_file(net, store, &out_path, MAX_RESULT_BYTES as u64).await + crate::pieces::publish_result_file(net, store, &job.accept, &out_path, MAX_RESULT_BYTES as u64).await } /// `ftyp` at offset 4 is the ISO-BMFF marker. The runner already ffprobed the @@ -803,6 +808,7 @@ mod tests { fn job() -> VideoJobInline { VideoJobInline { + image_key: None, model: "wan2.2-ti2v-5b".into(), prompt: "a neon alley in the rain".into(), negative_prompt: String::new(), diff --git a/node/crates/c0mpute-core/src/video_cpu.rs b/node/crates/c0mpute-core/src/video_cpu.rs index 8c8c47b6..b2e9f30a 100644 --- a/node/crates/c0mpute-core/src/video_cpu.rs +++ b/node/crates/c0mpute-core/src/video_cpu.rs @@ -483,7 +483,7 @@ async fn run_job(net: &Libp2pNetwork, store: &ChunkStore, cache_root: &Path, job bail!("video result is not an MP4"); } } - crate::pieces::publish_result_file(net, store, &out, video::MAX_RESULT_BYTES as u64).await + crate::pieces::publish_result_file(net, store, &job.accept, &out, video::MAX_RESULT_BYTES as u64).await } #[cfg(test)] diff --git a/node/crates/c0mpute-core/src/webhook.rs b/node/crates/c0mpute-core/src/webhook.rs index ebecbd19..8a35cc6a 100644 --- a/node/crates/c0mpute-core/src/webhook.rs +++ b/node/crates/c0mpute-core/src/webhook.rs @@ -591,12 +591,18 @@ pub fn pricing(price_usd: f64) -> PricingConfig { /// Bid only on valid webhook offers whose cap covers our price. pub fn offer_filter(price_usd: f64) -> OfferFilter { Arc::new(move |offer| { + // On an e2e offer the client key rides sealed; the budget is public. + let spec = match crate::job_e2e::bid_spec(offer) { + Some(crate::job_e2e::BidSpec::Public(v)) => { + Some(crate::job_e2e::with_placeholders(v, &[("client_pubkey", serde_json::json!("11".repeat(32)))])) + } + Some(crate::job_e2e::BidSpec::Full(v)) => Some(v.clone()), + None => None, + }; offer.max_price_usd.is_finite() && offer.max_price_usd >= price_usd - && offer - .spec_inline - .as_ref() - .and_then(|v| serde_json::from_value::(v.clone()).ok()) + && spec + .and_then(|v| serde_json::from_value::(v).ok()) .is_some_and(|s| { let deadline_ok = offer.deadline_unix_ms.saturating_sub(offer.published_at_ms) / 1000 >= s.budget_secs(); s.validate().is_ok() && deadline_ok @@ -1201,6 +1207,7 @@ mod tests { external_ref: None, buyer_addrs: Vec::new(), bid_window_ms: None, + e2e: None, } } @@ -1229,8 +1236,8 @@ mod tests { #[test] fn accepts_must_match_the_webhook_offer_we_bid_on() { let s = serde_json::to_value(spec(3)).unwrap(); - let bid = crate::dispatch::OpenBid { cap: 0.002, price: 0.001, buyer_peer_id: "buyer".into(), spec_inline: Some(s.clone()), external_ref: Some("coinpay:inv_1".into()), expires_at_ms: u64::MAX / 2, reserve_until_ms: u64::MAX / 2, payload: None }; - let accept = JobAccept { job_id: "j".into(), buyer_peer_id: "buyer".into(), winning_bidder_peer_id: "us".into(), agreed_price_usd: 0.001, workload_type: WORKLOAD.into(), spec_inline: Some(s), published_at_ms: 0, external_ref: Some("coinpay:inv_1".into()), buyer_addrs: Vec::new() }; + let bid = crate::dispatch::OpenBid { cap: 0.002, price: 0.001, buyer_peer_id: "buyer".into(), spec_inline: Some(s.clone()), external_ref: Some("coinpay:inv_1".into()), expires_at_ms: u64::MAX / 2, reserve_until_ms: u64::MAX / 2, payload: None, e2e: None }; + let accept = JobAccept { job_id: "j".into(), buyer_peer_id: "buyer".into(), winning_bidder_peer_id: "us".into(), agreed_price_usd: 0.001, workload_type: WORKLOAD.into(), spec_inline: Some(s), published_at_ms: 0, external_ref: Some("coinpay:inv_1".into()), buyer_addrs: Vec::new(), e2e: None, result_key: None }; assert!(bid.matches(&accept, 0).is_ok()); // A forged accept swapping in another buyer key (to read the call) is refused. let mut a = accept.clone(); diff --git a/node/crates/c0mpute-core/src/workflow.rs b/node/crates/c0mpute-core/src/workflow.rs index b876b08b..74c0ec8b 100644 --- a/node/crates/c0mpute-core/src/workflow.rs +++ b/node/crates/c0mpute-core/src/workflow.rs @@ -599,7 +599,7 @@ async fn run_step_job( let spec = resolve_spec(net, job).await?; let result = execute(&spec).await; let bytes = serde_json::to_vec(&result)?; - crate::pieces::publish_result_bytes(net, store, &bytes) + crate::pieces::publish_result_bytes(net, store, &job.accept, &bytes) .await .context("publish step result") } diff --git a/node/crates/c0mpute-core/tests/e2e_jobs_e2e.rs b/node/crates/c0mpute-core/tests/e2e_jobs_e2e.rs new file mode 100644 index 00000000..b6dde32b --- /dev/null +++ b/node/crates/c0mpute-core/tests/e2e_jobs_e2e.rs @@ -0,0 +1,455 @@ +//! End-to-end encrypted jobs (DIP-0032) on loopback libp2p: a buyer, two +//! workers and a passive node that subscribes to every job topic and records +//! everything gossip shows it. +//! +//! For every workload: the passive node sees no prompt, input or result; the +//! winner receives the exact spec; the loser cannot fetch the sealed spec; +//! the result chunk is ciphertext to anyone but the buyer; the buyer opens it. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use c0mpute_core::runner::AcceptedJob; +use c0mpute_core::{JobAuction, dispatch, gateway_util, job_e2e, pieces, run_auction}; +use c0mpute_net::topics::{JobAccept, JobReceipt, JobStatus, job_topic}; +use c0mpute_net::{ChunkSource, Libp2pNetwork, NetworkConfig, PeerId}; +use c0mpute_proto::Hash; +use c0mpute_store::ChunkStore; +use libp2p::multiaddr::Protocol; +use serde_json::{Value, json}; +use tempfile::{TempDir, tempdir}; +use tokio::sync::mpsc; + +struct StoreSource(ChunkStore); + +#[async_trait::async_trait] +impl ChunkSource for StoreSource { + async fn read_chunk(&self, hash: &Hash) -> anyhow::Result> { + self.0.get(hash).await + } +} + +struct Node { + net: Arc, + store: ChunkStore, + _dir: TempDir, +} + +async fn node(bootstrap: Vec) -> Node { + let dir = tempdir().unwrap(); + let store = ChunkStore::open(dir.path().join("store")).await.unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.path().join("id")) + .with_listen(vec!["/ip4/127.0.0.1/tcp/0".parse().unwrap()]) + .with_local_source(Arc::new(StoreSource(store.clone()))) + .with_bootstrap(bootstrap), + ) + .await + .unwrap(); + Node { + net: Arc::new(net), + store, + _dir: dir, + } +} + +async fn addr(net: &Libp2pNetwork) -> libp2p::Multiaddr { + for _ in 0..50 { + if let Ok(addrs) = net.listen_addrs().await + && let Some(a) = addrs.into_iter().next() + { + return a.with(Protocol::P2p(net.peer_id())); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("no listen addr"); +} + +/// One spec per workload, every sensitive field marked `SECRET`. +fn specs() -> Vec<(&'static str, Value)> { + vec![ + ( + "tts.synthesize", + json!({"text": "SECRET tts line", "engine": "kokoro", "voice": "af_heart", "locale": "en-US", "speed": 1.0}), + ), + ( + "diffusion.txt2img", + json!({"prompt": "SECRET castle", "model": "sd-turbo", "width": 512, "height": 512, "steps": 2}), + ), + ( + "music.txt2music", + json!({"prompt": "SECRET song", "lyrics": "SECRET verse", "duration": 10}), + ), + ( + "video.txt2vid", + json!({"model": "m", "prompt": "SECRET clip", "negative_prompt": "SECRET no", "frames": 8}), + ), + ( + "embed.text", + json!({"v": 1, "model": "bge-small-en-v1.5", "texts": ["SECRET document one"]}), + ), + ( + "ocr.extract", + json!({"v": 1, "input_hash": "SECRET", "format": "png", "lang": "eng"}), + ), + ( + "ffmpeg.transcode", + json!({"input_url": "https://SECRET.example/a.mp4", "preset": "720p", "spec": {"codec": "h264", "bitrate_bps": 1, "width": 2, "height": 2, "keyframe_interval": 1, "extra_ffmpeg_args": ["drawtext=SECRET"]}}), + ), + ( + "blender.render-frame", + json!({"v": 1, "input_hash": "SECRET", "input_size": 5, "input_pieces": ["SECRET"], "frame_start": 1, "frame_end": 1}), + ), + ( + "crawl.fetch", + json!({"v": 1, "url": "https://SECRET.example/", "headers": {"cookie": "SECRET"}}), + ), + ( + "workflow.step", + json!({"v": 1, "kind": "http_call", "config": {"url": "https://SECRET.example/"}, "input": {"note": "SECRET"}}), + ), + ( + "infernet.text.train", + json!({"job": {"kind": "lora", "base_model": "org/m", "system_prompt": "SECRET"}, "dataset_hash": "SECRET"}), + ), + ( + "infernet.inference", + json!({"v": 1, "model": "qwen2.5:0.5b", "messages": [{"role": "user", "content": "SECRET question"}]}), + ), + ( + "infernet.text.infer", + json!({"adapter_hash": "SECRET", "base_model": "org/m", "messages": [{"role": "user", "content": "SECRET"}], "max_new_tokens": 8}), + ), + ( + "captcha.solve", + json!({"v": 1, "kind": "text", "image": "SECRET"}), + ), + ] +} + +const RESULT: &[u8] = b"{\"ok\":true,\"text\":\"RESULT-SECRET answer\"}"; +const INPUT: &[u8] = b"INPUT-SECRET audio bytes"; + +/// A worker for `workload` whose runner records the spec it got, checks a +/// sealed input when the spec names one, seals `RESULT` back and publishes a +/// receipt. +fn serve(w: &Node, workload: &str, got: Arc>>) { + let (tx, mut rx) = mpsc::channel::(8); + dispatch::run_worker_subscriber(w.net.clone(), workload.to_string(), vec![], Some(tx)); + let (net, store) = (w.net.clone(), w.store.clone()); + tokio::spawn(async move { + while let Some(job) = rx.recv().await { + let spec = job.accept.spec_inline.clone().unwrap(); + if let Some(h) = spec["audio_hash"].as_str() { + let key = pieces::input_key(spec["audio_key"].as_str()).unwrap(); + let r = pieces::InputRef::parse("audio", h, None, None, 1 << 20).unwrap(); + let buyer: PeerId = job.accept.buyer_peer_id.parse().unwrap(); + let plain = r + .fetch_opened(&net, Some(buyer), 1 << 20, key.as_ref()) + .await + .unwrap(); + assert_eq!(plain, INPUT, "the winner opens the sealed input"); + } + got.lock().unwrap().push((job.accept.job_id.clone(), spec)); + // Long enough for the loser's attempt on the sealed spec. + tokio::time::sleep(Duration::from_millis(800)).await; + let hash = pieces::publish_result_bytes(&net, &store, &job.accept, RESULT) + .await + .unwrap(); + receipt(&net, &job.accept, hash).await; + } + }); +} + +async fn receipt(net: &Libp2pNetwork, accept: &JobAccept, hash: String) { + let r = JobReceipt { + job_id: accept.job_id.clone(), + worker_peer_id: net.peer_id().to_base58(), + worker_did: None, + buyer_peer_id: accept.buyer_peer_id.clone(), + output_hash: Some(hash), + status: JobStatus::Completed, + completed_at_ms: 0, + signature: None, + external_ref: None, + }; + let topic = job_topic(&accept.workload_type); + let bytes = serde_json::to_vec(&r).unwrap(); + let buyer: PeerId = accept.buyer_peer_id.parse().unwrap(); + let _ = net + .send_direct(buyer, &topic, &bytes, Duration::from_secs(5)) + .await; + let _ = net.publish(&topic, bytes).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn gossip_shows_no_plaintext_and_only_the_winner_reads_the_job() { + // SAFETY: set before any node starts; the test owns its process env. + unsafe { std::env::set_var("C0MPUTE_DISABLE_MDNS", "1") }; + let _ = tracing_subscriber::fmt() + .with_env_filter("c0mpute_core=info,c0mpute_net=warn") + .with_test_writer() + .try_init(); + + let a = node(vec![]).await; + let a_addr = addr(&a.net).await; + let b = node(vec![a_addr.clone()]).await; + let passive = node(vec![a_addr.clone()]).await; + let buyer = node(vec![a_addr.clone(), addr(&b.net).await]).await; + + // The passive node joins every job topic and keeps every byte it sees. + let seen: Arc>>> = Arc::default(); + let accepts: Arc>> = Arc::default(); + let got: Arc>> = Arc::default(); + for (w, _) in specs() { + passive.net.subscribe(&job_topic(w)).await.unwrap(); + serve(&a, w, got.clone()); + serve(&b, w, got.clone()); + } + { + let (seen, accepts) = (seen.clone(), accepts.clone()); + let mut rx = passive.net.messages(); + tokio::spawn(async move { + while let Ok(m) = rx.recv().await { + if let Ok(acc) = serde_json::from_slice::(&m.data) + && !acc.winning_bidder_peer_id.is_empty() + { + accepts.lock().unwrap().push(acc); + } + seen.lock().unwrap().push(m.data); + } + }); + } + tokio::time::sleep(Duration::from_secs(2)).await; + + let workers = vec![a.net.peer_id().to_base58(), b.net.peer_id().to_base58()]; + let mut cases = specs(); + // stt carries its audio as a sealed input stored on the buyer. + let (audio, audio_key) = + pieces::put_bytes_pieces_sealed(&buyer.net, &buyer.store, INPUT, pieces::INPUT_PIECE_BYTES) + .await + .unwrap(); + let mut stt = json!({"audio_provider_peer_id": buyer.net.peer_id().to_base58(), "audio_format": "wav", "model": "base"}); + stt.as_object_mut() + .unwrap() + .extend(audio.sealed_spec_fields("audio", &audio_key)); + cases.push(("stt.transcribe", stt)); + serve(&a, "stt.transcribe", got.clone()); + serve(&b, "stt.transcribe", got.clone()); + passive + .net + .subscribe(&job_topic("stt.transcribe")) + .await + .unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + + for (workload, spec) in cases { + // The loser tries to fetch the sealed spec as soon as the accept is + // gossiped; the buyer serves it only to the winner. + let loser_try = { + let (accepts, a_net, b_net, buyer_peer) = ( + accepts.clone(), + a.net.clone(), + b.net.clone(), + buyer.net.peer_id(), + ); + let workload = workload.to_string(); + tokio::spawn(async move { + for _ in 0..100 { + let acc = accepts + .lock() + .unwrap() + .iter() + .find(|x| x.workload_type == workload) + .cloned(); + if let Some(acc) = acc { + let loser = if acc.winning_bidder_peer_id == a_net.peer_id().to_base58() { + b_net + } else { + a_net + }; + return Some( + loser + .rpc( + buyer_peer, + job_e2e::PAYLOAD_SERVICE, + acc.job_id.into_bytes(), + ) + .await, + ); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + None + }) + }; + let auction = JobAuction::new(workload, spec.clone()) + .with_allowed_bidders(workers.clone()) + .with_max_price_usd(0.01) + .with_bid_window(Duration::from_secs(2)) + .with_receipt_timeout(Duration::from_secs(30)); + let outcome = run_auction(buyer.net.clone(), auction) + .await + .unwrap_or_else(|e| panic!("{workload}: {e:#}")); + assert_eq!(outcome.receipt.status, JobStatus::Completed, "{workload}"); + + // The winner got exactly the spec; nobody else got it. + let jobs: Vec<_> = got + .lock() + .unwrap() + .iter() + .filter(|(id, _)| *id == outcome.job_id) + .cloned() + .collect(); + assert_eq!(jobs.len(), 1, "{workload}: exactly one worker ran the job"); + assert_eq!( + jobs[0].1, spec, + "{workload}: the winner opened the exact spec" + ); + + // The loser was refused the sealed spec. + let refused = loser_try.await.unwrap().expect("accept seen on gossip"); + assert!( + refused.is_err(), + "{workload}: the loser fetched the sealed spec" + ); + + // The buyer opens the result. + let opened = + gateway_util::fetch_result(&buyer.net, &outcome, 1 << 20, Duration::from_secs(20)) + .await + .unwrap(); + assert_eq!(opened, RESULT, "{workload}"); + + // Anyone else who fetches the result chunk gets ciphertext. + let hash = outcome.receipt.output_hash.clone().unwrap(); + let winner = outcome.accepted_bid.bidder_peer_id.clone(); + let stolen = gateway_util::fetch_stored_result( + &passive.net, + &hash, + &winner, + 1 << 20, + Duration::from_secs(20), + ) + .await + .unwrap(); + assert!( + !contains(&stolen, b"RESULT-SECRET"), + "{workload}: result chunk is plaintext" + ); + assert!(job_e2e::open_blob(&job_e2e::random_key(), &stolen).is_err()); + } + + // The sealed stt input is ciphertext to the passive node too. + let stolen = gateway_util::fetch_stored_result( + &passive.net, + &audio.hash, + &buyer.net.peer_id().to_base58(), + 1 << 20, + Duration::from_secs(20), + ) + .await + .unwrap(); + assert!(!contains(&stolen, b"INPUT-SECRET")); + + // Everything gossip carried: offers, bids, accepts, receipts. No secret. + let seen = seen.lock().unwrap(); + assert!( + seen.len() >= 3 * 15, + "the passive node saw the auctions ({} messages)", + seen.len() + ); + for m in seen.iter() { + for needle in [ + &b"SECRET"[..], + b"RESULT-SECRET", + b"INPUT-SECRET", + b"tts line", + b"castle", + b"question", + ] { + assert!( + !contains(m, needle), + "gossip leaked {:?}: {}", + String::from_utf8_lossy(needle), + String::from_utf8_lossy(m) + ); + } + } + // And it did see what bidding needs. + assert!( + seen.iter().any(|m| contains(m, b"qwen2.5:0.5b")), + "public spec carries the model" + ); +} + +fn contains(hay: &[u8], needle: &[u8]) -> bool { + hay.windows(needle.len()).any(|w| w == needle) +} + +/// Live check (DIP-0032): join the real network as a passive node, subscribe +/// to the job topics, record every gossip message for +/// `C0MPUTE_CAPTURE_SECS` (default 120) into `C0MPUTE_CAPTURE_FILE`, and fail +/// if any contains `C0MPUTE_CAPTURE_NEEDLE`. Run it while a real job with +/// that needle in its prompt goes through: +/// `cargo test -p c0mpute-core --test e2e_jobs_e2e live_capture -- --ignored`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "joins the live network"] +async fn live_capture_shows_no_plaintext() { + let needle = std::env::var("C0MPUTE_CAPTURE_NEEDLE").expect("C0MPUTE_CAPTURE_NEEDLE"); + let secs: u64 = std::env::var("C0MPUTE_CAPTURE_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(120); + let out = std::env::var("C0MPUTE_CAPTURE_FILE").unwrap_or_else(|_| "capture.jsonl".into()); + let dir = tempdir().unwrap(); + let net = Libp2pNetwork::spawn( + NetworkConfig::for_dir(dir.path().join("id")) + .with_listen(vec!["/ip4/0.0.0.0/tcp/0".parse().unwrap()]) + .with_bootstrap(c0mpute_net::bootstrap::default_seeds()), + ) + .await + .unwrap(); + let topics = [ + "infernet.inference", + "diffusion.txt2img", + "tts.synthesize", + "stt.transcribe", + "embed.text", + "infernet.text.infer", + ]; + for t in topics { + net.subscribe(&job_topic(t)).await.unwrap(); + } + let mut rx = net.messages(); + let mut f = std::fs::File::create(&out).unwrap(); + let (mut n, mut leaks, mut offers) = (0usize, 0usize, 0usize); + let deadline = tokio::time::Instant::now() + Duration::from_secs(secs); + while let Ok(Ok(m)) = tokio::time::timeout_at(deadline, rx.recv()).await { + n += 1; + if contains(&m.data, needle.as_bytes()) { + leaks += 1; + } + if let Ok(o) = serde_json::from_slice::(&m.data) { + offers += 1; + eprintln!( + "offer {} {} e2e={} spec_inline={}", + o.workload_type, + o.job_id, + o.e2e.is_some(), + o.spec_inline.is_some() + ); + } + use std::io::Write; + writeln!( + f, + "{}\t{}\t{}", + m.topic, + m.source.map(|p| p.to_base58()).unwrap_or_default(), + String::from_utf8_lossy(&m.data) + ) + .unwrap(); + } + eprintln!("captured {n} messages ({offers} offers) on {topics:?}; {leaks} contain the needle"); + assert_eq!(leaks, 0, "plaintext on gossip"); +} diff --git a/node/crates/c0mpute-core/tests/gameserver_e2e.rs b/node/crates/c0mpute-core/tests/gameserver_e2e.rs index bcb46d82..ca59a090 100644 --- a/node/crates/c0mpute-core/tests/gameserver_e2e.rs +++ b/node/crates/c0mpute-core/tests/gameserver_e2e.rs @@ -266,6 +266,7 @@ async fn secrets_are_sealed_and_backups_readable_only_by_the_buyer() { /// from that backup (handed to B under a fresh key). #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn watch_fails_over_to_another_worker_from_the_latest_backup() { + let _ = tracing_subscriber::fmt().with_env_filter(std::env::var("RUST_LOG").unwrap_or_else(|_| "warn,c0mpute_core::buyer=info,c0mpute_core::dispatch=info,c0mpute_core::gameserver=info,c0mpute_core::handoff=debug".into())).with_test_writer().try_init(); let tmp = tempdir().unwrap(); let rt_a = Fake::new(world(11)); let rt_b = Fake::new(world(99)); diff --git a/node/crates/c0mpute-core/tests/large_result_e2e.rs b/node/crates/c0mpute-core/tests/large_result_e2e.rs index 8924b9d0..43070b9b 100644 --- a/node/crates/c0mpute-core/tests/large_result_e2e.rs +++ b/node/crates/c0mpute-core/tests/large_result_e2e.rs @@ -102,7 +102,7 @@ async fn a_12_mib_result_reaches_the_buyer() { // Before: one chunk. The fetch fails on the codec's response cap. let whole = worker_store.put(&result).await.unwrap(); worker.announce(&whole).await.unwrap(); - let single = gateway_util::fetch_result( + let single = gateway_util::fetch_stored_result( &buyer, &whole.to_hex(), &provider, @@ -116,11 +116,11 @@ async fn a_12_mib_result_reaches_the_buyer() { ); // After: pieces plus a manifest; the receipt names the manifest. - let receipt_hash = pieces::publish_result_bytes(&worker, &worker_store, &result) + let receipt_hash = pieces::publish_stored_bytes(&worker, &worker_store, &result) .await .unwrap(); assert_ne!(receipt_hash, Hash::of(&result).to_hex()); - let got = gateway_util::fetch_result( + let got = gateway_util::fetch_stored_result( &buyer, &receipt_hash, &provider, @@ -133,7 +133,7 @@ async fn a_12_mib_result_reaches_the_buyer() { assert!(got == result, "reassembled result differs"); // The buyer's cap still holds for a pieced result. - let over = gateway_util::fetch_result( + let over = gateway_util::fetch_stored_result( &buyer, &receipt_hash, &provider, @@ -169,7 +169,7 @@ async fn small_and_legacy_results_stay_single_chunk() { // At the threshold: one chunk, receipt hash = content hash. let small = noise(b"{\"text\":\"", pieces::RESULT_SINGLE_CHUNK_MAX); - let h = pieces::publish_result_bytes(&worker, &worker_store, &small) + let h = pieces::publish_stored_bytes(&worker, &worker_store, &small) .await .unwrap(); assert_eq!(h, Hash::of(&small).to_hex()); @@ -179,24 +179,24 @@ async fn small_and_legacy_results_stay_single_chunk() { .unwrap(); assert!(old_buyer == small); let new_buyer = - gateway_util::fetch_result(&buyer, &h, &provider, 10 * MIB, Duration::from_secs(20)) + gateway_util::fetch_stored_result(&buyer, &h, &provider, 10 * MIB, Duration::from_secs(20)) .await .unwrap(); assert!(new_buyer == small); // A single chunk over the buyer's cap is refused. assert!(matches!( - gateway_util::fetch_result(&buyer, &h, &provider, 1024, Duration::from_secs(20)).await, + gateway_util::fetch_stored_result(&buyer, &h, &provider, 1024, Duration::from_secs(20)).await, Err(FetchError::Invalid(_)) )); // One byte over: pieced. let two = noise(b"{", pieces::RESULT_SINGLE_CHUNK_MAX + 1); - let h = pieces::publish_result_bytes(&worker, &worker_store, &two) + let h = pieces::publish_stored_bytes(&worker, &worker_store, &two) .await .unwrap(); assert_ne!(h, Hash::of(&two).to_hex()); assert!( - gateway_util::fetch_result(&buyer, &h, &provider, 10 * MIB, Duration::from_secs(30)) + gateway_util::fetch_stored_result(&buyer, &h, &provider, 10 * MIB, Duration::from_secs(30)) .await .unwrap() == two @@ -206,7 +206,7 @@ async fn small_and_legacy_results_stay_single_chunk() { let legacy = wav(4 * MIB); let lh = worker_store.put(&legacy).await.unwrap(); worker.announce(&lh).await.unwrap(); - let got = gateway_util::fetch_result( + let got = gateway_util::fetch_stored_result( &buyer, &lh.to_hex(), &provider, @@ -242,7 +242,7 @@ async fn a_forged_manifest_is_refused() { }; let mh = worker_store.put(&manifest.encode()).await.unwrap(); worker.announce(&mh).await.unwrap(); - let got = gateway_util::fetch_result( + let got = gateway_util::fetch_stored_result( &buyer, &mh.to_hex(), &provider, @@ -277,18 +277,18 @@ async fn file_round_trip(len: usize) { } // Over the worker's cap is refused before anything is stored. assert!( - pieces::publish_result_file(&worker, &worker_store, &src, len as u64 - 1) + pieces::publish_stored_file(&worker, &worker_store, &src, len as u64 - 1) .await .is_err() ); let started = Instant::now(); - let h = pieces::publish_result_file(&worker, &worker_store, &src, len as u64) + let h = pieces::publish_stored_file(&worker, &worker_store, &src, len as u64) .await .unwrap(); let stored = started.elapsed(); let dest = tmp.path().join("got.mp4"); let started = Instant::now(); - let written = gateway_util::fetch_result_to_file( + let written = gateway_util::fetch_stored_result_to_file( &buyer, &h, &provider, diff --git a/node/crates/c0mpute-core/tests/text_train_e2e.rs b/node/crates/c0mpute-core/tests/text_train_e2e.rs index 922b938e..91d0ef5a 100644 --- a/node/crates/c0mpute-core/tests/text_train_e2e.rs +++ b/node/crates/c0mpute-core/tests/text_train_e2e.rs @@ -145,7 +145,7 @@ async fn trains_peer_to_peer_with_pieces_and_conversations() { // A tampered manifest hash is refused (nothing is written). let bogus = c0mpute_proto::Hash::of(b"not a manifest").to_hex(); - let err = text::fetch_adapter_pieces(&buyer, Some(w.net.peer_id()), &bogus, &tmp.path().join("bad")).await; + let err = text::fetch_adapter_pieces(&buyer, Some(w.net.peer_id()), &bogus, &tmp.path().join("bad"), None).await; assert!(err.is_err()); assert!(!tmp.path().join("bad").exists()); @@ -168,9 +168,11 @@ async fn trains_peer_to_peer_with_pieces_and_conversations() { assert_eq!(ans["text"], "[HuggingFaceTB/SmolLM2-135M-Instruct] system=You are a pricing coach. | user=How should I price it?", "{ans}"); assert_eq!(ans["adapter_hash"], out["adapter_hash"]); assert!(!ans["receipt"]["output_hash"].as_str().unwrap().starts_with("sha256:"), "answer came back as a chunk"); - // The worker cached the adapter it fetched as pieces. + // The adapter came sealed (DIP-0032): the worker kept it only for the + // job, never in its adapter cache, and removed it afterwards. let cached = tmp.path().join("w/cache/text/adapters").join(out["adapter_hash"].as_str().unwrap().trim_start_matches("sha256:")); - assert_eq!(std::fs::read(cached.join("adapter_model.safetensors")).unwrap(), want); + assert!(!cached.join("adapter_model.safetensors").exists(), "a sealed adapter must not be cached"); + let _ = &want; } /// A corpus with conversations is never placed on a worker whose runner diff --git a/node/crates/c0mpute-net/Cargo.toml b/node/crates/c0mpute-net/Cargo.toml index 839174b5..39a657a2 100644 --- a/node/crates/c0mpute-net/Cargo.toml +++ b/node/crates/c0mpute-net/Cargo.toml @@ -15,6 +15,7 @@ async-trait = { workspace = true } futures = { workspace = true } libp2p = { workspace = true } serde_json = { workspace = true } +zeroize = { workspace = true } reqwest = { workspace = true } [dev-dependencies] diff --git a/node/crates/c0mpute-net/src/identity.rs b/node/crates/c0mpute-net/src/identity.rs index 2e605a12..4d20e6dd 100644 --- a/node/crates/c0mpute-net/src/identity.rs +++ b/node/crates/c0mpute-net/src/identity.rs @@ -77,10 +77,35 @@ pub fn ed25519_public_hex(peer: &libp2p::PeerId) -> Option { Some(ed.to_bytes().iter().map(|b| format!("{b:02x}")).collect()) } +/// Check that `sig` is `peer`'s signature over `msg`, using the public key +/// the peer id inlines (identity multihash). Peer ids that hash their key +/// (RSA) cannot be checked this way and never verify. +pub fn verify_peer_signature(peer: &libp2p::PeerId, msg: &[u8], sig: &[u8]) -> bool { + let mh: &libp2p::multihash::Multihash<64> = peer.as_ref(); + if mh.code() != 0x00 { + return false; + } + let Ok(public) = libp2p::identity::PublicKey::try_decode_protobuf(mh.digest()) else { + return false; + }; + public.to_peer_id() == *peer && public.verify(msg, sig) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn peer_signatures_verify_only_for_the_signer() { + let kp = Keypair::generate_ed25519(); + let other = Keypair::generate_ed25519().public().to_peer_id(); + let peer = kp.public().to_peer_id(); + let sig = kp.sign(b"hello").unwrap(); + assert!(verify_peer_signature(&peer, b"hello", &sig)); + assert!(!verify_peer_signature(&peer, b"hellp", &sig)); + assert!(!verify_peer_signature(&other, b"hello", &sig)); + } + #[test] fn peer_id_yields_its_ed25519_key() { let kp = Keypair::generate_ed25519(); diff --git a/node/crates/c0mpute-net/src/swarm.rs b/node/crates/c0mpute-net/src/swarm.rs index ed58f11b..741d9f6b 100644 --- a/node/crates/c0mpute-net/src/swarm.rs +++ b/node/crates/c0mpute-net/src/swarm.rs @@ -236,6 +236,9 @@ pub struct Libp2pNetwork { gossip_tx: broadcast::Sender, peer_id: PeerId, rpc_handlers: RpcHandlers, + /// The node identity, kept to sign statements other peers check against + /// our peer id ([`Self::sign`]). + keypair: identity::Keypair, } impl Libp2pNetwork { @@ -250,7 +253,7 @@ impl Libp2pNetwork { let rpc_handlers: RpcHandlers = Arc::new(std::sync::RwLock::new(HashMap::new())); let task = SwarmTask::new( - keypair, + keypair.clone(), config.local_source, gossip_tx.clone(), rpc_handlers.clone(), @@ -288,9 +291,16 @@ impl Libp2pNetwork { gossip_tx, peer_id, rpc_handlers, + keypair, }) } + /// Sign `msg` with this node's identity key. Anyone can check it against + /// our peer id with [`crate::identity::verify_peer_signature`]. + pub fn sign(&self, msg: &[u8]) -> Result> { + self.keypair.sign(msg).map_err(|e| anyhow!("sign: {e}")) + } + /// Serve `service` over `/c0mpute/rpc/1.0.0`. Replaces any handler /// already registered under the name. pub fn register_rpc(&self, service: &str, handler: Arc) { diff --git a/node/crates/c0mpute-net/src/topics.rs b/node/crates/c0mpute-net/src/topics.rs index 2c8952fb..386c7bf2 100644 --- a/node/crates/c0mpute-net/src/topics.rs +++ b/node/crates/c0mpute-net/src/topics.rs @@ -85,11 +85,82 @@ pub fn valid_external_ref(s: &str) -> bool { !s.is_empty() && s.len() <= MAX_EXTERNAL_REF_LEN && s.bytes().all(|b| b.is_ascii_graphic()) } +// ──────────────────────────────────────────────────────────────────────── +// End-to-end encrypted jobs (DIP-0032) +// ──────────────────────────────────────────────────────────────────────── +// +// Gossip topics are readable by every subscriber and relay. An e2e offer +// carries no spec, only what bidding needs (`OfferE2e::public_spec`) and the +// buyer's fresh X25519 key. A bid carries the worker's fresh X25519 key, +// signed by the worker's libp2p identity. After the award the buyer seals +// the spec to the winner alone and hands it over point to point +// (request-response); the result comes back sealed to the buyer. + +/// Version of the e2e job envelope this build speaks. +pub const JOB_E2E_VERSION: u8 = 1; + +/// Capability tag every e2e-capable worker satisfies. Buyers list it in +/// `required_capabilities`, so a worker older than e2e (which checks that it +/// advertises every required tag) never bids on an e2e offer. +pub const JOB_E2E_TAG: &str = "c0mpute:e2e:v1"; + +/// Capability tag for a worker running inside an attested TEE (AMD SEV-SNP, +/// Intel TDX). An offer that requires it only matches workers that attach +/// attestation evidence to their bid; see [`BidE2e::attestation`]. +pub const CONFIDENTIAL_TAG: &str = "c0mpute:confidential"; + +/// The e2e part of a [`JobOffer`]. +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct OfferE2e { + /// [`JOB_E2E_VERSION`]. + pub v: u8, + /// The buyer's X25519 public key for this one auction, lowercase hex. + pub buyer_key: String, + /// What a worker needs to decide whether to bid (model, engine, size + /// buckets...). Never prompts, inputs, URLs or file names. + #[serde(default)] + pub public_spec: serde_json::Value, + /// Present when the buyer requires an attested TEE: the challenge the + /// worker's attestation report must bind (with its X25519 key). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attest_nonce: Option, +} + +/// The e2e part of a [`JobBid`]. +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct BidE2e { + /// The worker's X25519 public key for this one job, lowercase hex. + pub worker_key: String, + /// The worker's libp2p identity signature (base64) over the binding of + /// `worker_key` to this job, buyer and buyer key. The buyer checks it + /// against `bidder_peer_id` before sealing anything to the key. + pub sig: String, + /// Remote attestation evidence (format named inside), for offers that + /// require [`CONFIDENTIAL_TAG`]. Its report data binds `worker_key`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub attestation: Option, +} + +/// A 32-byte job key held in memory only: never serialized, never printed, +/// wiped on drop. +#[derive(Clone, PartialEq, Eq)] +pub struct JobKey(pub [u8; 32]); + +impl std::fmt::Debug for JobKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("JobKey()") + } +} + +impl Drop for JobKey { + fn drop(&mut self) { + zeroize::Zeroize::zeroize(&mut self.0); + } +} + /// Buyer publishes this to `c0mpute/jobs/` to start the -/// auction. The actual job spec is opaque to the network — workers -/// evaluate by `workload_type` + `spec_hash` + price/deadline. Inputs + -/// secrets are referenced by URL or content hash inside `spec`, -/// optionally end-to-end encrypted. +/// auction. An e2e offer (`e2e` set, `spec_inline` empty) shows the network +/// only what bidding needs; see the e2e section above. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub struct JobOffer { /// UUID minted by the buyer. @@ -147,6 +218,10 @@ pub struct JobOffer { /// Optional; old buyers omit it. #[serde(default, skip_serializing_if = "Option::is_none")] pub bid_window_ms: Option, + /// End-to-end encryption (DIP-0032). When set, `spec_inline` is empty + /// and `spec_hash` is a salted commitment to the spec, not its hash. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub e2e: Option, } /// Worker bid in response to a `JobOffer`. Sent on the same job topic; @@ -177,6 +252,9 @@ pub struct JobBid { /// Buyers skip overloaded workers when a free one bids. #[serde(default, skip_serializing_if = "Option::is_none")] pub load: Option, + /// The worker's signed X25519 key, on bids for e2e offers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub e2e: Option, } impl JobBid { @@ -213,6 +291,14 @@ pub struct JobAccept { /// buyer usually cannot dial a worker's random, firewalled port. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub buyer_addrs: Vec, + /// [`JOB_E2E_VERSION`] when the job is e2e: `spec_inline` is empty and + /// the winner fetches the sealed spec from the buyer point to point. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub e2e: Option, + /// Worker side only, set once the sealed spec is opened: the key the + /// result is sealed to the buyer with. Never serialized. + #[serde(skip)] + pub result_key: Option, } /// Worker's completion receipt. Includes the output hash so the buyer @@ -273,6 +359,7 @@ mod tests { expires_at_ms: Some(1000), bench_score: None, load: None, + e2e: None, }; let json = serde_json::to_value(&bid).unwrap(); assert_eq!(json["expires_at_ms"], 1000); @@ -317,6 +404,7 @@ mod tests { expires_at_ms: None, bench_score: None, load: None, + e2e: None, }; assert!(bid.is_live(u64::MAX), "old bids never expire"); bid.expires_at_ms = Some(100); diff --git a/package.json b/package.json index a009ddf0..194b53da 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute-monorepo", - "version": "0.3.2", + "version": "0.3.3", "private": true, "workspaces": [ "apps/*", diff --git a/packages/captcha/package.json b/packages/captcha/package.json index 8fe8d340..a56da320 100644 --- a/packages/captcha/package.json +++ b/packages/captcha/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/c0mpute-captcha", - "version": "0.3.2", + "version": "0.3.3", "description": "Solve image, slider, gap, click-text, audio and math captchas on the c0mpute network: library (fetch().then(solveCaptcha)), CLI and stdio MCP server.", "type": "module", "license": "MIT", diff --git a/packages/shared/package.json b/packages/shared/package.json index 6b880379..977271ef 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/shared", - "version": "0.3.2", + "version": "0.3.3", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/plugins/ad-generator-agent/module.toml b/plugins/ad-generator-agent/module.toml index 3069e090..b568267e 100644 --- a/plugins/ad-generator-agent/module.toml +++ b/plugins/ad-generator-agent/module.toml @@ -21,7 +21,7 @@ [module] id = "ad-generator-agent" name = "Ad Generator Agent — every ad type from one URL" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "applications" status = "available" diff --git a/plugins/ads-manager/module.toml b/plugins/ads-manager/module.toml index 0e5d467f..67b2a370 100644 --- a/plugins/ads-manager/module.toml +++ b/plugins/ads-manager/module.toml @@ -38,7 +38,7 @@ [module] id = "ads-manager" name = "Ads Manager — shared ad operations layer" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "applications" status = "partial" diff --git a/plugins/apps/module.toml b/plugins/apps/module.toml index daf7d3d7..917d4c32 100644 --- a/plugins/apps/module.toml +++ b/plugins/apps/module.toml @@ -18,7 +18,7 @@ [module] id = "apps" name = "Apps — backends with 3x redundancy" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/captcha/module.toml b/plugins/captcha/module.toml index 51c6dc82..c4744ad1 100644 --- a/plugins/captcha/module.toml +++ b/plugins/captcha/module.toml @@ -22,7 +22,7 @@ [module] id = "captcha" name = "Captcha — image, slider, audio and math solving" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/coinpay/module.toml b/plugins/coinpay/module.toml index a0321991..0835b7c0 100644 --- a/plugins/coinpay/module.toml +++ b/plugins/coinpay/module.toml @@ -7,7 +7,7 @@ [module] id = "coinpay" name = "CoinPay — DID, payments, escrow, reputation" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "applications" status = "available" diff --git a/plugins/crawl/module.toml b/plugins/crawl/module.toml index 5d569155..040fc128 100644 --- a/plugins/crawl/module.toml +++ b/plugins/crawl/module.toml @@ -8,7 +8,7 @@ [module] id = "crawl" name = "Crawl — distributed web fetching" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "data" status = "available" diff --git a/plugins/crowdgpt/module.toml b/plugins/crowdgpt/module.toml index 6c2f7e1b..5ce5a63d 100644 --- a/plugins/crowdgpt/module.toml +++ b/plugins/crowdgpt/module.toml @@ -43,7 +43,7 @@ [module] id = "crowdgpt" name = "CrowdGPT: donate idle compute" -version = "0.3.2" +version = "0.3.3" kind = "donation" category = "ai" status = "available" diff --git a/plugins/ctv-ads/module.toml b/plugins/ctv-ads/module.toml index 25a72dea..8a847f7d 100644 --- a/plugins/ctv-ads/module.toml +++ b/plugins/ctv-ads/module.toml @@ -28,7 +28,7 @@ [module] id = "ctv-ads" name = "CTV Ads — VAST and VMAP from every gateway" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "applications" status = "partial" diff --git a/plugins/diffusion/module.toml b/plugins/diffusion/module.toml index cb00125c..0cf0f90b 100644 --- a/plugins/diffusion/module.toml +++ b/plugins/diffusion/module.toml @@ -11,7 +11,7 @@ [module] id = "diffusion" name = "Diffusion — image generation" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/embed/module.toml b/plugins/embed/module.toml index 9bb07049..0bf28568 100644 --- a/plugins/embed/module.toml +++ b/plugins/embed/module.toml @@ -19,7 +19,7 @@ [module] id = "embed" name = "Embeddings — vector generation" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index 0f4e8ef3..25f2d513 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -55,7 +55,7 @@ [module] id = "gameserver" name = "Game server — Minecraft / Factorio / Valheim" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/hosting/module.toml b/plugins/hosting/module.toml index 5882e920..f532979e 100644 --- a/plugins/hosting/module.toml +++ b/plugins/hosting/module.toml @@ -30,7 +30,7 @@ [module] id = "hosting" name = "Hosting — p2p static sites" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "data" status = "available" diff --git a/plugins/infernet/module.toml b/plugins/infernet/module.toml index c50fc180..a32dc85a 100644 --- a/plugins/infernet/module.toml +++ b/plugins/infernet/module.toml @@ -22,7 +22,7 @@ [module] id = "infernet" name = "Infernet Protocol — AI inference" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/live-stream/module.toml b/plugins/live-stream/module.toml index 33272997..49c55c76 100644 --- a/plugins/live-stream/module.toml +++ b/plugins/live-stream/module.toml @@ -34,7 +34,7 @@ [module] id = "live-stream" name = "Live Stream — p2p HLS live streaming" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "media" status = "available" diff --git a/plugins/music/module.toml b/plugins/music/module.toml index 7c3eaf75..f4dd8831 100644 --- a/plugins/music/module.toml +++ b/plugins/music/module.toml @@ -15,7 +15,7 @@ [module] id = "music" name = "Music — song generation (ACE-Step 1.5)" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/ocr/module.toml b/plugins/ocr/module.toml index 83f468f3..8489a61f 100644 --- a/plugins/ocr/module.toml +++ b/plugins/ocr/module.toml @@ -19,7 +19,7 @@ [module] id = "ocr" name = "OCR — document text extraction" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/podcast-ads/module.toml b/plugins/podcast-ads/module.toml index b2a39c44..208feff1 100644 --- a/plugins/podcast-ads/module.toml +++ b/plugins/podcast-ads/module.toml @@ -31,7 +31,7 @@ [module] id = "podcast-ads" name = "Podcast Ads — dynamic ad insertion" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "applications" status = "partial" diff --git a/plugins/podcasting/module.toml b/plugins/podcasting/module.toml index 7f18d327..d0f2828a 100644 --- a/plugins/podcasting/module.toml +++ b/plugins/podcasting/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasting" name = "Podcasting — p2p podcast publishing" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "media" status = "available" diff --git a/plugins/podcasts/module.toml b/plugins/podcasts/module.toml index 27769c10..818a6df7 100644 --- a/plugins/podcasts/module.toml +++ b/plugins/podcasts/module.toml @@ -24,7 +24,7 @@ [module] id = "podcasts" name = "Podcasts — p2p podcast listener" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "media" status = "available" diff --git a/plugins/render/module.toml b/plugins/render/module.toml index a215ce8b..7b08f016 100644 --- a/plugins/render/module.toml +++ b/plugins/render/module.toml @@ -33,7 +33,7 @@ [module] id = "render" name = "Render — Blender 3D rendering" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "media" status = "available" diff --git a/plugins/secure-chat/module.toml b/plugins/secure-chat/module.toml index 4731d0ea..831fe7cb 100644 --- a/plugins/secure-chat/module.toml +++ b/plugins/secure-chat/module.toml @@ -21,7 +21,7 @@ [module] id = "secure-chat" name = "Secure Chat — E2E encrypted p2p messaging" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "applications" status = "available" diff --git a/plugins/storage/module.toml b/plugins/storage/module.toml index 79c044b2..8038f1d2 100644 --- a/plugins/storage/module.toml +++ b/plugins/storage/module.toml @@ -20,7 +20,7 @@ [module] id = "storage" name = "Storage — Reed-Solomon shard service" -version = "0.3.2" +version = "0.3.3" kind = "service" category = "data" status = "available" diff --git a/plugins/train/module.toml b/plugins/train/module.toml index ac9822f0..a4dbef90 100644 --- a/plugins/train/module.toml +++ b/plugins/train/module.toml @@ -55,7 +55,7 @@ [module] id = "train" name = "Fine-tune — LoRA training" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/transcode/module.toml b/plugins/transcode/module.toml index b65f9610..2862e0cf 100644 --- a/plugins/transcode/module.toml +++ b/plugins/transcode/module.toml @@ -7,7 +7,7 @@ [module] id = "transcode" name = "FFmpeg Transcoding" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "media" status = "available" diff --git a/plugins/tts/module.toml b/plugins/tts/module.toml index 0d8ddf1e..64d6de73 100644 --- a/plugins/tts/module.toml +++ b/plugins/tts/module.toml @@ -15,7 +15,7 @@ [module] id = "tts" name = "TTS — text-to-speech" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/video/module.toml b/plugins/video/module.toml index 52d1db82..b89c26c7 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -65,7 +65,7 @@ [module] id = "video" name = "Video — generation (Wan 2.2, Wan 2.1 1.3B on 8 GiB or CPU, LTX-2.3 with audio)" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/webhook/module.toml b/plugins/webhook/module.toml index 6d65c5ee..ecce8571 100644 --- a/plugins/webhook/module.toml +++ b/plugins/webhook/module.toml @@ -7,7 +7,7 @@ [module] id = "webhook" name = "Webhooks — paid, end-to-end encrypted delivery" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/whisper/module.toml b/plugins/whisper/module.toml index 056831a9..2eaf5846 100644 --- a/plugins/whisper/module.toml +++ b/plugins/whisper/module.toml @@ -7,7 +7,7 @@ [module] id = "whisper" name = "Whisper — speech-to-text" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "ai" status = "available" diff --git a/plugins/workflow-step/module.toml b/plugins/workflow-step/module.toml index b8086cbd..574ff1f5 100644 --- a/plugins/workflow-step/module.toml +++ b/plugins/workflow-step/module.toml @@ -14,7 +14,7 @@ [module] id = "workflow-step" name = "Workflow step — webhook workflow steps (meshhook)" -version = "0.3.2" +version = "0.3.3" kind = "workload" category = "infrastructure" status = "available"