From 623b722714b5506b17b239f181be43e1eeb5aab0 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 8 Nov 2026 18:22:20 +0000 Subject: [PATCH] fix(diffusion): kill sd-cli renders orphaned by an in-place update; bump v0.2.69 Live on seed2: each auto-update re-exec left the running sd-cli alive (same PID, so still our child, result never read). It held ~6.5 GB and the CPU, so the node fell under the 12 GB free-RAM gate and stopped bidding, or drew the next image at half speed (559 s / 873 s vs 107 s clean). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 32 +++++----- Cargo.toml | 2 +- apps/tui/package.json | 2 +- apps/web/package.json | 2 +- .../web/src/app/releases/latest.json/route.ts | 2 +- node/crates/c0mpute-core/src/diffusion.rs | 64 +++++++++++++++++++ package.json | 2 +- packages/shared/package.json | 2 +- plugins/ad-generator-agent/module.toml | 2 +- plugins/ads-manager/module.toml | 2 +- plugins/coinpay/module.toml | 2 +- plugins/crawl/module.toml | 2 +- plugins/ctv-ads/module.toml | 2 +- plugins/diffusion/module.toml | 2 +- plugins/embed/module.toml | 2 +- plugins/gameserver/module.toml | 2 +- plugins/hosting/module.toml | 2 +- plugins/infernet/module.toml | 2 +- plugins/live-stream/module.toml | 2 +- plugins/ocr/module.toml | 2 +- plugins/podcast-ads/module.toml | 2 +- plugins/podcasting/module.toml | 2 +- plugins/podcasts/module.toml | 2 +- plugins/render/module.toml | 2 +- plugins/secure-chat/module.toml | 2 +- plugins/storage/module.toml | 2 +- plugins/train/module.toml | 2 +- plugins/transcode/module.toml | 2 +- plugins/tts/module.toml | 2 +- plugins/video/module.toml | 2 +- plugins/webhook/module.toml | 2 +- plugins/whisper/module.toml | 2 +- plugins/workflow-step/module.toml | 2 +- 33 files changed, 111 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 73df895c..bfa44b22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -478,7 +478,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "c0mpute-api" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "c0mpute-proto", @@ -493,7 +493,7 @@ dependencies = [ [[package]] name = "c0mpute-bench" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "blake3", @@ -504,7 +504,7 @@ dependencies = [ [[package]] name = "c0mpute-cli" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "axum", @@ -534,7 +534,7 @@ dependencies = [ [[package]] name = "c0mpute-core" -version = "0.2.68" +version = "0.2.69" dependencies = [ "aes-gcm", "anyhow", @@ -578,7 +578,7 @@ dependencies = [ [[package]] name = "c0mpute-doctor" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "serde", @@ -590,7 +590,7 @@ dependencies = [ [[package]] name = "c0mpute-envelope" -version = "0.2.68" +version = "0.2.69" dependencies = [ "blake3", "bs58", @@ -603,7 +603,7 @@ dependencies = [ [[package]] name = "c0mpute-gateway" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "axum", @@ -625,7 +625,7 @@ dependencies = [ [[package]] name = "c0mpute-market" -version = "0.2.68" +version = "0.2.69" dependencies = [ "c0mpute-envelope", "serde", @@ -635,7 +635,7 @@ dependencies = [ [[package]] name = "c0mpute-net" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "async-trait", @@ -652,7 +652,7 @@ dependencies = [ [[package]] name = "c0mpute-placement" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "async-trait", @@ -673,7 +673,7 @@ dependencies = [ [[package]] name = "c0mpute-proto" -version = "0.2.68" +version = "0.2.69" dependencies = [ "blake3", "hex", @@ -685,7 +685,7 @@ dependencies = [ [[package]] name = "c0mpute-secure-chat" -version = "0.2.68" +version = "0.2.69" dependencies = [ "aes-gcm", "anyhow", @@ -708,7 +708,7 @@ dependencies = [ [[package]] name = "c0mpute-store" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "blake3", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "c0mpute-transcode" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "c0mpute-proto", @@ -739,7 +739,7 @@ dependencies = [ [[package]] name = "c0mpute-update" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "hex", @@ -753,7 +753,7 @@ dependencies = [ [[package]] name = "c0mpute-verify" -version = "0.2.68" +version = "0.2.69" dependencies = [ "anyhow", "blake3", diff --git a/Cargo.toml b/Cargo.toml index b6f6508d..ad2f4b8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ ] [workspace.package] -version = "0.2.68" +version = "0.2.69" edition = "2024" license = "MIT" repository = "https://github.com/profullstack/c0mpute" diff --git a/apps/tui/package.json b/apps/tui/package.json index 61928f77..20af29f9 100644 --- a/apps/tui/package.json +++ b/apps/tui/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/tui", - "version": "0.2.68", + "version": "0.2.69", "private": true, "type": "module", "bin": { diff --git a/apps/web/package.json b/apps/web/package.json index e79f0881..d6f0c692 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute", - "version": "0.2.68", + "version": "0.2.69", "private": true, "type": "module", "scripts": { diff --git a/apps/web/src/app/releases/latest.json/route.ts b/apps/web/src/app/releases/latest.json/route.ts index 80188eba..e3cf4e21 100644 --- a/apps/web/src/app/releases/latest.json/route.ts +++ b/apps/web/src/app/releases/latest.json/route.ts @@ -28,7 +28,7 @@ export const dynamic = "force-dynamic"; * this in sync with the workspace version in `Cargo.toml` on each release so a * freshly built CLI reports "already latest" rather than a phantom upgrade. */ -const FALLBACK_VERSION = "0.2.68"; +const FALLBACK_VERSION = "0.2.69"; const FALLBACK_MIN_REQUIRED = "0.0.1"; type Channel = "stable" | "beta" | "nightly"; diff --git a/node/crates/c0mpute-core/src/diffusion.rs b/node/crates/c0mpute-core/src/diffusion.rs index 4fc986f6..b79bd88d 100644 --- a/node/crates/c0mpute-core/src/diffusion.rs +++ b/node/crates/c0mpute-core/src/diffusion.rs @@ -509,12 +509,69 @@ fn node_slot() -> Arc { SLOT.get_or_init(|| Arc::new(Semaphore::new(1))).clone() } +/// `/proc//stat` -> (state, parent pid). The command name is in +/// parentheses and may contain spaces, so parse after the last `)`. +fn parse_stat(stat: &str) -> Option<(char, u32)> { + let rest = stat.get(stat.rfind(')')? + 1..)?.trim_start(); + let mut fields = rest.split_whitespace(); + let state = fields.next()?.chars().next()?; + let ppid = fields.next()?.parse().ok()?; + Some((state, ppid)) +} + +/// Kill renders left running by the binary this process replaced. +/// +/// An auto-update re-executes the worker in place: same PID, so a running +/// sd-cli stays our child but nothing will ever read its result, and it +/// keeps ~6-9 GB and every core busy, so the node stops bidding (free-RAM +/// gate) or runs the next image at half speed. Called once when the runner +/// starts, before it takes any job, so every sd-cli child is such an orphan. +pub fn kill_orphaned_renders() -> usize { + if !cfg!(target_os = "linux") { + return 0; + } + let me = std::process::id(); + let ours = sd_cli(); + let Ok(entries) = std::fs::read_dir("/proc") else { + return 0; + }; + let mut killed = 0; + for entry in entries.flatten() { + let name = entry.file_name(); + let Some(pid) = name.to_str().and_then(|s| s.parse::().ok()) else { + continue; + }; + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + continue; + }; + let Some((state, ppid)) = parse_stat(&stat) else { + continue; + }; + if ppid != me || state == 'Z' { + continue; + } + if std::fs::read_link(format!("/proc/{pid}/exe")).ok().as_deref() != Some(ours.as_path()) { + continue; + } + if std::process::Command::new("kill") + .args(["-9", &pid.to_string()]) + .status() + .is_ok_and(|s| s.success()) + { + warn!(pid, "killed an sd-cli render orphaned by an in-place update"); + killed += 1; + } + } + killed +} + pub fn spawn_diffusion_runner( net: Arc, store: ChunkStore, cache_root: PathBuf, load: WorkerLoad, ) -> mpsc::Sender { + kill_orphaned_renders(); let (tx, mut rx) = mpsc::channel::(4); tokio::spawn(async move { while let Some(job) = rx.recv().await { @@ -644,6 +701,13 @@ mod tests { ); } + #[test] + fn proc_stat_parses_state_and_parent_even_with_spaces_in_the_name() { + assert_eq!(parse_stat("3655695 (sd-cli) R 4165541 3655695 0"), Some(('R', 4165541))); + assert_eq!(parse_stat("12 (a b) c) Z 7 12 0"), Some(('Z', 7))); + assert_eq!(parse_stat("garbage"), None); + } + #[test] fn png_magic_is_checked() { assert!(is_png(b"\x89PNG\r\n\x1a\n0000")); diff --git a/package.json b/package.json index d106ceac..39a45243 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "c0mpute-monorepo", - "version": "0.2.68", + "version": "0.2.69", "private": true, "workspaces": [ "apps/*", diff --git a/packages/shared/package.json b/packages/shared/package.json index 1302a19c..8b7d7451 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@c0mpute/shared", - "version": "0.2.68", + "version": "0.2.69", "private": true, "type": "module", "main": "./dist/index.js", diff --git a/plugins/ad-generator-agent/module.toml b/plugins/ad-generator-agent/module.toml index 69f1b92e..b787771e 100644 --- a/plugins/ad-generator-agent/module.toml +++ b/plugins/ad-generator-agent/module.toml @@ -21,7 +21,7 @@ [module] id = "ad-generator-agent" name = "Ad Generator Agent — every ad type from one URL" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "applications" status = "available" diff --git a/plugins/ads-manager/module.toml b/plugins/ads-manager/module.toml index 1c2ca503..58c8f925 100644 --- a/plugins/ads-manager/module.toml +++ b/plugins/ads-manager/module.toml @@ -18,7 +18,7 @@ [module] id = "ads-manager" name = "Ads Manager — shared ad operations layer" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "applications" status = "coming-soon" diff --git a/plugins/coinpay/module.toml b/plugins/coinpay/module.toml index 99611e46..022a2427 100644 --- a/plugins/coinpay/module.toml +++ b/plugins/coinpay/module.toml @@ -7,7 +7,7 @@ [module] id = "coinpay" name = "CoinPay — DID, payments, escrow, reputation" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "applications" status = "available" diff --git a/plugins/crawl/module.toml b/plugins/crawl/module.toml index fd78651e..e8fcda29 100644 --- a/plugins/crawl/module.toml +++ b/plugins/crawl/module.toml @@ -8,7 +8,7 @@ [module] id = "crawl" name = "Crawl — distributed web fetching" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "data" status = "available" diff --git a/plugins/ctv-ads/module.toml b/plugins/ctv-ads/module.toml index 6a3dfade..8adc3ef5 100644 --- a/plugins/ctv-ads/module.toml +++ b/plugins/ctv-ads/module.toml @@ -20,7 +20,7 @@ [module] id = "ctv-ads" name = "CTV Ads — Connected TV ad marketplace" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "applications" status = "coming-soon" diff --git a/plugins/diffusion/module.toml b/plugins/diffusion/module.toml index 7b0c05de..9bc71ff5 100644 --- a/plugins/diffusion/module.toml +++ b/plugins/diffusion/module.toml @@ -11,7 +11,7 @@ [module] id = "diffusion" name = "Diffusion — image generation" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "ai" status = "available" diff --git a/plugins/embed/module.toml b/plugins/embed/module.toml index efd41cdc..ec5dda8d 100644 --- a/plugins/embed/module.toml +++ b/plugins/embed/module.toml @@ -13,7 +13,7 @@ [module] id = "embed" name = "Embeddings — vector generation" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "ai" status = "coming-soon" diff --git a/plugins/gameserver/module.toml b/plugins/gameserver/module.toml index a5710a20..18ee8ad2 100644 --- a/plugins/gameserver/module.toml +++ b/plugins/gameserver/module.toml @@ -14,7 +14,7 @@ [module] id = "gameserver" name = "Game server — Minecraft / Factorio / Valheim" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "infrastructure" status = "coming-soon" diff --git a/plugins/hosting/module.toml b/plugins/hosting/module.toml index 4308543f..074393d1 100644 --- a/plugins/hosting/module.toml +++ b/plugins/hosting/module.toml @@ -12,7 +12,7 @@ [module] id = "hosting" name = "Hosting — censorship-resistant static sites" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "data" status = "coming-soon" diff --git a/plugins/infernet/module.toml b/plugins/infernet/module.toml index 1262bc1c..cccb287a 100644 --- a/plugins/infernet/module.toml +++ b/plugins/infernet/module.toml @@ -10,7 +10,7 @@ [module] id = "infernet" name = "Infernet Protocol — AI inference" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "ai" status = "partial" diff --git a/plugins/live-stream/module.toml b/plugins/live-stream/module.toml index 57fe0e47..a8e9b4bf 100644 --- a/plugins/live-stream/module.toml +++ b/plugins/live-stream/module.toml @@ -23,7 +23,7 @@ [module] id = "live-stream" name = "Live Stream — p2p HLS live streaming" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "media" status = "coming-soon" diff --git a/plugins/ocr/module.toml b/plugins/ocr/module.toml index 698720c7..7814667f 100644 --- a/plugins/ocr/module.toml +++ b/plugins/ocr/module.toml @@ -12,7 +12,7 @@ [module] id = "ocr" name = "OCR — document text extraction" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "ai" status = "coming-soon" diff --git a/plugins/podcast-ads/module.toml b/plugins/podcast-ads/module.toml index 8b12189f..bfd4bec1 100644 --- a/plugins/podcast-ads/module.toml +++ b/plugins/podcast-ads/module.toml @@ -22,7 +22,7 @@ [module] id = "podcast-ads" name = "Podcast Ads — audio ad marketplace" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "applications" status = "coming-soon" diff --git a/plugins/podcasting/module.toml b/plugins/podcasting/module.toml index da57ae3d..3e46a93b 100644 --- a/plugins/podcasting/module.toml +++ b/plugins/podcasting/module.toml @@ -20,7 +20,7 @@ [module] id = "podcasting" name = "Podcasting — p2p podcast publishing" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "media" status = "coming-soon" diff --git a/plugins/podcasts/module.toml b/plugins/podcasts/module.toml index 8511440a..8f313be3 100644 --- a/plugins/podcasts/module.toml +++ b/plugins/podcasts/module.toml @@ -19,7 +19,7 @@ [module] id = "podcasts" name = "Podcasts — p2p podcast listener" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "media" status = "coming-soon" diff --git a/plugins/render/module.toml b/plugins/render/module.toml index f7dfc7b7..f84386b7 100644 --- a/plugins/render/module.toml +++ b/plugins/render/module.toml @@ -13,7 +13,7 @@ [module] id = "render" name = "Render — Blender 3D rendering" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "media" status = "coming-soon" diff --git a/plugins/secure-chat/module.toml b/plugins/secure-chat/module.toml index 5b3bb25d..aa0df69a 100644 --- a/plugins/secure-chat/module.toml +++ b/plugins/secure-chat/module.toml @@ -18,7 +18,7 @@ [module] id = "secure-chat" name = "Secure Chat — E2E encrypted p2p messaging" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "applications" status = "partial" diff --git a/plugins/storage/module.toml b/plugins/storage/module.toml index d47b9552..22c86d22 100644 --- a/plugins/storage/module.toml +++ b/plugins/storage/module.toml @@ -20,7 +20,7 @@ [module] id = "storage" name = "Storage — Reed-Solomon shard service" -version = "0.2.68" +version = "0.2.69" kind = "service" category = "data" status = "available" diff --git a/plugins/train/module.toml b/plugins/train/module.toml index 0193ab32..19b4517e 100644 --- a/plugins/train/module.toml +++ b/plugins/train/module.toml @@ -13,7 +13,7 @@ [module] id = "train" name = "Fine-tune — LoRA/QLoRA training" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "ai" status = "coming-soon" diff --git a/plugins/transcode/module.toml b/plugins/transcode/module.toml index f050ef8c..23c4e00e 100644 --- a/plugins/transcode/module.toml +++ b/plugins/transcode/module.toml @@ -7,7 +7,7 @@ [module] id = "transcode" name = "FFmpeg Transcoding" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "media" status = "available" diff --git a/plugins/tts/module.toml b/plugins/tts/module.toml index 12e46b6d..34de01f4 100644 --- a/plugins/tts/module.toml +++ b/plugins/tts/module.toml @@ -15,7 +15,7 @@ [module] id = "tts" name = "TTS — text-to-speech" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "ai" status = "available" diff --git a/plugins/video/module.toml b/plugins/video/module.toml index f8fbe4d2..89a03a1e 100644 --- a/plugins/video/module.toml +++ b/plugins/video/module.toml @@ -44,7 +44,7 @@ [module] id = "video" name = "Video — generation (Wan 2.2, LTX-2.3 with audio)" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "ai" status = "partial" diff --git a/plugins/webhook/module.toml b/plugins/webhook/module.toml index 7214f259..86f54d66 100644 --- a/plugins/webhook/module.toml +++ b/plugins/webhook/module.toml @@ -7,7 +7,7 @@ [module] id = "webhook" name = "Webhooks — paid, end-to-end encrypted delivery" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "infrastructure" status = "available" diff --git a/plugins/whisper/module.toml b/plugins/whisper/module.toml index d38553ac..3dc34928 100644 --- a/plugins/whisper/module.toml +++ b/plugins/whisper/module.toml @@ -7,7 +7,7 @@ [module] id = "whisper" name = "Whisper — speech-to-text" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "ai" status = "available" diff --git a/plugins/workflow-step/module.toml b/plugins/workflow-step/module.toml index 13bb7dfc..818a0aae 100644 --- a/plugins/workflow-step/module.toml +++ b/plugins/workflow-step/module.toml @@ -14,7 +14,7 @@ [module] id = "workflow-step" name = "Workflow step — webhook workflow steps (meshhook)" -version = "0.2.68" +version = "0.2.69" kind = "workload" category = "infrastructure" status = "available"