From f298533d86dd1ff4986e1efd1196cb52e5305117 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:10:00 +0000 Subject: [PATCH 1/5] fix(qualification): detect hash-only rescue by tensor absence, not any file Hash-only rescue carries job.json and summary.json, so the "no file under published/" test sent real hash-only trees down the full-rescue branch and rejected them with qualification_artifact_custody_changed. --- src/hypertrain/gpu_ops/network_qualification.py | 5 ++++- tests/gpu_ops/test_network_gpu_qualification.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hypertrain/gpu_ops/network_qualification.py b/src/hypertrain/gpu_ops/network_qualification.py index 698b380..272fc13 100644 --- a/src/hypertrain/gpu_ops/network_qualification.py +++ b/src/hypertrain/gpu_ops/network_qualification.py @@ -487,8 +487,11 @@ def compare(paths: list[Path], manifest: RunManifestV2) -> dict[str, bool]: raise Reject("qualification_two_host_binding_or_bitwise_mismatch") expected = set(round_files(ref.layout.n_gpus, inner.H, inner.J)) for path, result in zip(paths, (a, b), strict=True): + # Hash-only rescue still carries JSON (job.json, summary.json): only tensors decide. hash_only = not any( - p.is_file() or p.is_symlink() for p in path.parent.glob("round-*/published/**/*") + (p := path.parent / f"round-{index}/published/{rel}").is_file() or p.is_symlink() + for index in range(len(result.rounds)) + for rel in expected ) claimed = host_inventory(path) if hash_only else {} for index, files in enumerate(result.rounds): diff --git a/tests/gpu_ops/test_network_gpu_qualification.py b/tests/gpu_ops/test_network_gpu_qualification.py index e0caecd..9c29bb5 100644 --- a/tests/gpu_ops/test_network_gpu_qualification.py +++ b/tests/gpu_ops/test_network_gpu_qualification.py @@ -134,6 +134,11 @@ def hash_only(paths: list[Path]) -> None: for p in root.glob("round-*/published/**/*"): if p.is_file(): p.unlink() + # The rescue carries per-rank summaries and job bindings (HASH_RESCUE_NAMES). + for rank in root.glob("round-*/published/rank-*"): + (rank / "summary.json").write_text('{"backend": "cuda"}') + for published in root.glob("round-*/published"): + (published / "job.json").write_text("{}") def test_hash_only_rescue_compares_from_bound_host_inventories(tmp_path: Path) -> None: From ee7aa3c04b8e3fe48d2220f4e62486f6f08aba81 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Thu, 15 Oct 2026 16:26:29 +0000 Subject: [PATCH 2/5] fix(launcher): no silent tree fallback, stale-root preflight, disjoint rescue ports, pinned miner host key - die when the frozen tree is missing instead of using /data/hypertrain-tree - refuse existing /data/hypertrain-h0/h1 before any spend (materialize would refuse an hour in) - h1 rescue base port 18600: 16 workers bind base..base+15, 18570 overlapped h0 - record launcher.pid: abort is TERM on it (setsid ignores INT/HUP) - default h1 miner ssh uses the admission-pinned known_hosts with StrictHostKeyChecking=yes --- scripts/launch_cuda_run.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/launch_cuda_run.sh b/scripts/launch_cuda_run.sh index fcdcab8..50c3311 100755 --- a/scripts/launch_cuda_run.sh +++ b/scripts/launch_cuda_run.sh @@ -64,14 +64,14 @@ healthy() { mexec python -c 'import httpx;assert httpx.get("http://127.0.0.1:800 rescued() { "$PY" -c "import json,sys;sys.exit(0 if any(r.get('kind')=='network_rescued' and r.get('role')==sys.argv[2] for r in map(json.loads,open(sys.argv[1]))) else 1)" "$RUN/journal.jsonl" "$1"; } mssh() { ssh -i "$MINER_KEY" -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 \ - -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$STATE/miner_known_hosts" \ + -o StrictHostKeyChecking="${MINER_HOSTKEY_POLICY:-accept-new}" -o UserKnownHostsFile="${MINER_KNOWN_HOSTS:-$STATE/miner_known_hosts}" \ -p "${MINER_SSH#* }" "root@${MINER_SSH%% *}" "$@" } export PYTHONHOME=$RD/runtime/local CUDA_VISIBLE_DEVICES='' HOME=$RD # Controller tree: the frozen snapshot admission used (same launcher/orchestrate as every stage). HT_TREE=${HT_TREE:-$("$PY" -c 'import json,sys;print(json.load(open(sys.argv[1]))["snapshot"])' "$PREP/freeze/freeze.json")/tree} -[ -f "$HT_TREE/experiments/gpu_network_v2/orchestrate.py" ] || HT_TREE=/data/hypertrain-tree +[ -f "$HT_TREE/experiments/gpu_network_v2/orchestrate.py" ] || { echo "frozen tree missing: $HT_TREE" >&2; exit 1; } export HT_TREE export PYTHONPATH=$HT_TREE/src:$HT_TREE/scripts:$HT_TREE:/data/pydeps:$RD/runtime/venv/lib/python3.12/site-packages mkdir -p "$STATE" @@ -86,6 +86,10 @@ for f in "$T"/{prepare_record.py,task8-deps.py,rescue_parallel.py,task8-material "$RD/keys/vast.key" "$RD/keys/owner.seed" "$HOTKEY_SEED" "$PROXY_TOKEN" "$COORD_KEY" "$RD/dispatch.json"; do [ -e "$f" ] || die "missing $f" done +# materialize refuses existing roots: fail here, before any spend, not an hour into the run. +[ -e "$STATE/roots.done" ] || for r in h0 h1; do + [ ! -e "/data/hypertrain-$r" ] || die "/data/hypertrain-$r exists: archive it after checking no run reads it" +done RID=$(J "$OUT/manifest-envelope.json" "d['run_id']") [ "$(J "$JOB/job.json" "d['run_id']")" = "$RID" ] || die "job-$TAG binds another run than out-$TAG" [ "$(J "$OUT/profile.json" "(d['admission']['gpu_name'],d['admission']['gpus_per_host'],d['admission']['sm_count'])")" \ @@ -130,6 +134,8 @@ if [ ! -e "$STATE/window" ]; then echo "$A $CUT" > "$STATE/window" fi read -r A CUT < "$STATE/window" +# Abort = kill -TERM "$(cat $STATE/launcher.pid)": the supervisor tracks this pid (INT/HUP are ignored). +echo $$ > "$STATE/launcher.pid" STAGE=beacon # The master clock only advances on POST /v1/admin/beacon; the pusher dies on ReadTimeout while @@ -172,7 +178,7 @@ if stage rescue; then pids=() for r in h0 h1; do rescued "$r" && continue - port=18560; [ "$r" = h1 ] && port=18570 + port=18560; [ "$r" = h1 ] && port=18600 # 16 workers bind base..base+15 each as_runtime "$PY" "$T/rescue_parallel.py" --run-dir "$RUN" --role "$r" --via range --workers 16 \ --public-ip "$PUBLIC" --base-port "$port" > "$STATE/rescue-$r.out" 2>&1 & pids+=($!) @@ -265,6 +271,7 @@ if stage miner; then MINER_SSH=$("$PY" -c "import json,sys;r=[x for x in map(json.loads,open(sys.argv[1])) if x.get('kind')=='instance_ready' and x.get('role')=='h1'][-1];print(r['ssh_host'],r['ssh_port'])" "$RUN/journal.jsonl") \ || die "no h1 instance_ready in $RUN/journal.jsonl" MINER_KEY=${MINER_KEY:-$RUN/id_ed25519} + MINER_HOSTKEY_POLICY=yes MINER_KNOWN_HOSTS=$RUN/known_hosts-h1 # pinned at admission fi MINER_KEY=${MINER_KEY:-$RD/vast-m5/id_ed25519} echo "$MINER_SSH" > "$STATE/miner_ssh" From e6b278ae8d4dac1d368d62d8d86a09c0c809e232 Mon Sep 17 00:00:00 2001 From: hypertrain-bot Date: Thu, 15 Oct 2026 19:20:35 +0000 Subject: [PATCH 3/5] feat: bandwidth A-Z: hash-only rescue, theta-only miner, lazy trace, H=512, real dataset --- experiments/gpu_network_v2/orchestrate.py | 262 ++++++-- scripts/launch_cuda_run.sh | 21 +- scripts/ops/auditors.sh | 57 ++ scripts/ops/backend_bootstrap.py | 89 +++ scripts/ops/hf_seed.py | 104 +++ scripts/ops/live_operator.py | 292 +++++++++ scripts/ops/manifest/od_large_manifest.py | 606 ++++++++++++++++++ scripts/ops/prepare_record.py | 293 +++++++++ scripts/ops/prepare_runtime.py | 440 +++++++++++++ scripts/ops/probe_and_launch.py | 360 +++++++++++ scripts/ops/rescue_dedup.py | 290 +++++++++ scripts/ops/rescue_parallel.py | 473 ++++++++++++++ scripts/ops/task8-deps.py | 270 ++++++++ scripts/ops/task8-materialize-roots.py | 323 ++++++++++ src/hypertrain/auditor/island_bisect.py | 111 +++- src/hypertrain/challenge/store.py | 18 +- src/hypertrain/data/store.py | 44 +- src/hypertrain/datasets/build.py | 20 +- src/hypertrain/datasets/registry.json | 4 +- .../gpu_ops/network_qualification.py | 4 +- src/hypertrain/miner/cli.py | 87 ++- src/hypertrain/miner/core.py | 136 +++- src/hypertrain/miner/island_launch.py | 48 +- src/hypertrain/miner/island_worker.py | 77 ++- src/hypertrain/protocol/messages_v2.py | 37 +- src/hypertrain/trainer/island.py | 41 +- tests/auditor/test_island_lazy_window.py | 80 +++ tests/data/test_parts.py | 46 ++ tests/datasets/test_datasets.py | 25 + .../gpu_ops/test_network_gpu_qualification.py | 11 +- tests/gpu_ops/test_network_lifecycle.py | 85 ++- tests/gpu_ops/test_network_role_dispatch.py | 1 + tests/miner/test_client_network_v2.py | 126 ++++ tests/protocol/test_network_v2.py | 31 + 34 files changed, 4737 insertions(+), 175 deletions(-) create mode 100755 scripts/ops/auditors.sh create mode 100644 scripts/ops/backend_bootstrap.py create mode 100644 scripts/ops/hf_seed.py create mode 100644 scripts/ops/live_operator.py create mode 100644 scripts/ops/manifest/od_large_manifest.py create mode 100644 scripts/ops/prepare_record.py create mode 100644 scripts/ops/prepare_runtime.py create mode 100755 scripts/ops/probe_and_launch.py create mode 100644 scripts/ops/rescue_dedup.py create mode 100644 scripts/ops/rescue_parallel.py create mode 100644 scripts/ops/task8-deps.py create mode 100644 scripts/ops/task8-materialize-roots.py create mode 100644 tests/auditor/test_island_lazy_window.py diff --git a/experiments/gpu_network_v2/orchestrate.py b/experiments/gpu_network_v2/orchestrate.py index 9f203ff..fc703d0 100644 --- a/experiments/gpu_network_v2/orchestrate.py +++ b/experiments/gpu_network_v2/orchestrate.py @@ -224,15 +224,30 @@ def artifact_budget(profile: dict[str, Any]) -> Record: forward = inn["grad_accum"] * (5 + 6 * layers + 20 * layers) # Conservative: every gradient replicated; every parameter owned for all 3 state hooks. step_events = forward + tensors * 3 + tensors + tensors * 3 + 6 + tensors * 3 - trace_events = inn["H"] * step_events + (leaves + 1) * 9 * tensors + # Commit path traces window 1 only (determinism probe; disputes re-trace one window). + trace_events = inn["J"] * step_events + (leaves + 1) * 9 * tensors trace = trace_events * a["trace_bytes_per_event"] sample_count = inn["H"] * inn["micro_batch"] * inn["grad_accum"] * lay["n_gpus"] sample_bytes = sample_count * (m["seq_len"] + 1) * 2 - # Inputs remain at job root AND copied into published; proofs/job JSON in metadata budget. - inputs = 2 * (state + ef + header + sample_bytes) - outputs = lay["n_gpus"] * ((leaves + 1) * state + ef + delta + trace) + # Remote disk: inputs stay at job root AND are copied into published (two real copies); + # proofs/job JSON in metadata budget. Logs: two files, each under the ulimit file cap. + inputs_once = state + ef + header + sample_bytes + inputs = 2 * inputs_once + # Checkpoints are written by rank 0 only; every rank writes its final state/ef/delta/trace. + outputs = leaves * state + lay["n_gpus"] * (state + ef + delta + trace) logs = max(a["logs_bytes_per_execution"], 2 * math.ceil(max(trace, state) / 1024) * 1024) execution = inputs + outputs + logs + a["metadata_bytes_per_execution"] + # Dedup rescue carries each distinct sha256 once: ranks are bitwise identical except + # trace/summary, so one rank's tensors + every rank's trace, and inputs at most once. + transfer = ( + inputs_once + + (leaves + 1) * state + + ef + + delta + + lay["n_gpus"] * trace + + logs + + a["metadata_bytes_per_execution"] + ) policy = a.get("remote_retention", "all") if policy == "all": resident = profile["schedule"]["executions_per_host"] @@ -248,22 +263,20 @@ def artifact_budget(profile: dict[str, Any]) -> Record: raise Reject("unsupported_remote_retention_policy") retained = execution * resident # Tar record rounding + path headers + terminating records for every retained file. - files_per_execution = lay["n_gpus"] * (leaves + 5) + 14 + files_per_execution = lay["n_gpus"] * 6 + leaves + 14 tar_overhead = files_per_execution * resident * 2048 + 1048576 + 10240 - rescue = retained + tar_overhead + rescue = transfer * resident + tar_overhead per_host = profile["schedule"]["executions_per_host"] if policy == "all": remote_peak = retained + rescue # rescue.tar is written next to the outputs local = profile["schedule"]["hosts"] * (per_host + 1) * (rescue + execution) else: # The archive streams over SSH straight into local custody: no remote copy. Locally the - # qualification tar stays, and each workload keeps its tar plus one extraction (the - # canonical copy hardlinks the verified extraction; cross-rank dedup is not counted). + # qualification tar stays, and each workload keeps its tar plus one extraction whose + # identical files (other ranks, canonical copy) hardlink one verified copy. remote_peak = retained - workload_tar = execution + files_per_execution * 2048 + 1048576 + 10240 - local = profile["schedule"]["hosts"] * ( - rescue + (per_host - 2) * (workload_tar + execution) - ) + workload_tar = transfer + files_per_execution * 2048 + 1048576 + 10240 + local = profile["schedule"]["hosts"] * (rescue + (per_host - 2) * (workload_tar + transfer)) return { "remote_retention": policy, "local_custody_bytes_required": local, @@ -273,8 +286,11 @@ def artifact_budget(profile: dict[str, Any]) -> Record: "trace_events_per_rank": trace_events, "trace_bytes_per_rank": trace, "execution_bytes": execution, + "execution_transfer_bytes": transfer, "retained_bytes_per_host": retained, "rescue_tar_bytes_per_host": rescue, + # Legacy full-byte rescues (journal rows without dedup_known) re-verify against this. + "full_rescue_tar_bytes_per_host": retained + tar_overhead, "disk_artifacts_peak_bytes": remote_peak, "artifact_transfer_bytes_per_host": retained + rescue, "rescue_min_bytes_per_second": math.ceil(rescue / a["rescue_transfer_seconds"]), @@ -1047,7 +1063,7 @@ def operation( if prune: self.prune_rescued(role) bounds = artifact_budget(self.profile) - if shutil.disk_usage(self.lifecycle.dir).free < 2 * bounds["execution_bytes"]: + if shutil.disk_usage(self.lifecycle.dir).free < 2 * bounds["execution_transfer_bytes"]: raise Reject("local_custody_disk_exhausted") self.reserve(role, name, "workload", 1) self.journal.append( @@ -1139,8 +1155,14 @@ def stop() -> None: error = exc self.journal.append("network_operation_failed", role=role, name=name, error=str(exc)) finally: + # Inputs we uploaded travel back as hashes only: their bytes are already local. + local_inputs = { + fsha(p): p + for p in (directory / rel for rel in job.object_paths.values()) + if p.is_file() and not p.is_symlink() + } try: - custody = self.rescue(role, fresh=True) + custody = self.rescue(role, fresh=True, known=frozenset(local_inputs)) except ( Reject, OSError, @@ -1163,18 +1185,41 @@ def stop() -> None: destination = directory / name if destination.exists(): raise Reject("operation_result_destination_exists") + # Materialize every manifest file of this execution from one verified copy per sha256: + # a carried member (any path) or an input we uploaded. Ranks must match rank-0 first. + files: dict[str, Any] = custody["files"] + mine = {n.removeprefix(name + "/"): v for n, v in files.items() if n.startswith(name + "/")} + if rank_bytes_differ(mine): + raise Reject("operation_rank_bytes_differ") seen: dict[str, Path] = {} with tarfile.open(archive) as source: - for member in source.getmembers(): - prefix = "out/" + name + "/" - if member.name.startswith(prefix) and member.isfile(): - rel = member.name.removeprefix(prefix) - path = destination / rel - if Path(rel).is_absolute() or ".." in Path(rel).parts: - raise Reject("operation_custody_path") - stream = source.extractfile(member) + carried = { + files[m.name.removeprefix("out/")]["sha256"]: m + for m in source.getmembers() + if m.isfile() and m.name.removeprefix("out/") in files + } + for rel, meta in sorted(mine.items()): + path = destination / rel + if Path(rel).is_absolute() or ".." in Path(rel).parts: + raise Reject("operation_custody_path") + digest = meta["sha256"] + if digest in seen and prune: # verified by this extraction: hardlink + path.parent.mkdir(parents=True, exist_ok=True) + os.link(seen[digest], path) + continue + if digest in seen: + data = seen[digest].read_bytes() + elif digest in carried: + stream = source.extractfile(carried[digest]) assert stream is not None - place_verified(path, stream.read(), seen, prune) + data = stream.read() + elif digest in local_inputs: + data = local_inputs[digest].read_bytes() + else: + raise Reject("operation_custody_bytes_missing") + if sha256(data) != digest: + raise Reject("operation_custody_changed") + place_verified(path, data, seen, prune) result = json.loads((destination / "operation-result.json").read_bytes()) if ( result.get("status") != "CAPTURED_NOT_ACCEPTED" @@ -1582,17 +1627,7 @@ def completion_custody( ) if rescue is None or fsha(rescue["tar"]) != done["tar_sha256"]: raise Reject("completion_original_rescue_changed") - prefix = "out/" + name + "/" - with tarfile.open(rescue["tar"]) as archive: - rescued = {} - for member in archive.getmembers(): - if member.name.startswith(prefix) and member.isfile(): - rel = member.name.removeprefix(prefix) - if rel in rescued: - raise Reject("completion_duplicate_rescue_member") - stream = archive.extractfile(member) - assert stream is not None - rescued[rel] = sha256(stream.read()) + rescued = rescued_hashes(Path(rescue["tar"]), name + "/", "completion") expected_rescue = { publication_name + "/" + rel: meta["sha256"] for rel, meta in files.items() } @@ -1786,16 +1821,9 @@ def launch( ) if rescue is None or fsha(Path(rescue["tar"])) != done["tar_sha256"]: raise Reject("operation_canonical_rescue_changed") - prefix = "out/" + name + "/" + str(original.relative_to(custody)) + "/" - with tarfile.open(rescue["tar"]) as archive: - rescued = {} - for member in archive.getmembers(): - if member.name.startswith(prefix) and member.isfile(): - stream = archive.extractfile(member) - assert stream is not None - rescued[member.name.removeprefix(prefix)] = sha256(stream.read()) - if rescued != files: - raise Reject("operation_canonical_rescue_changed") + prefix = name + "/" + str(original.relative_to(custody)) + "/" + if rescued_hashes(Path(rescue["tar"]), prefix, "operation_canonical") != files: + raise Reject("operation_canonical_rescue_changed") canonical = directory / "published" with tempfile.TemporaryDirectory(prefix="canonical-", dir=directory) as temporary: staged = Path(temporary) / "published" @@ -2478,11 +2506,7 @@ def bound(r: Record, done: Record | None = done, qual: Record | None = qual) -> raise Reject("remote_prune_local_custody_missing") if fsha(archive) != custody["tar_sha256"]: raise Reject("remote_prune_local_custody_changed") - files = verify_export( - archive, - artifact_budget(self.profile)["rescue_tar_bytes_per_host"], - custody.get("hashes_only", False), - ) + files = verify_export(archive, *rescue_mode(self.profile, custody)) mine = [f for f in files if f.startswith(name + "/")] if files != custody["files"] or any( not any(f"rank-{r}" in Path(f).parts for f in mine) for r in ranks @@ -2501,8 +2525,11 @@ def bound(r: Record, done: Record | None = done, qual: Record | None = qual) -> ) return out - def rescue(self, role: str, fresh: bool = False) -> Record: - """Complete path-preserving, byte/hash checked custody; failure never becomes PASS.""" + def rescue(self, role: str, fresh: bool = False, known: frozenset[str] = frozenset()) -> Record: + """Complete path-preserving, byte/hash checked custody; failure never becomes PASS. + + known: sha256s whose bytes the caller already holds locally (uploaded inputs); after a + workload exists they travel as manifest hashes only, like every repeated content.""" generation = sha256( json.dumps( [ @@ -2526,11 +2553,7 @@ def rescue(self, role: str, fresh: bool = False) -> Record: archive = Path(old["tar"]) if not archive.is_file() or fsha(archive) != old["tar_sha256"]: raise Reject("network_rescue_changed_on_resume") - verify_export( - archive, - artifact_budget(self.profile)["rescue_tar_bytes_per_host"], - old.get("hashes_only", False), - ) + verify_export(archive, *rescue_mode(self.profile, old)) return old root = self.lifecycle.remote_root(role) ssh = self.lifecycle.ssh(role) @@ -2554,6 +2577,10 @@ def remaining() -> None: # Before any workload, out/ holds only seed + qualification: those rounds are proven by # the two hosts' independently computed hashes, so only hashes and small JSON travel. hashes_only = not self.journal.all("network_execution_intent", role=role, phase="workload") + # After a workload: every distinct content once (rank-0 bytes; identical ranks, inputs + # we hold, repeated checkpoints are manifest hashes only), never ~8x full ranks. + dedup = sorted(known) if not hashes_only else None + mode = " hashes" if hashes_only else " ".join([" dedup", *(dedup or [])]) if hashes_only: # Carried: per round job.json + n_gpus summaries; result/config/custody; seed # job.json; slack. Each <= 1 MiB + tar header, plus the <= 1 MiB manifest. @@ -2564,14 +2591,14 @@ def remaining() -> None: cmd = ( f"cd {shlex.quote(root)} && env PYTHONPATH=src {shlex.quote(python)} " "experiments/gpu_network_v2/orchestrate.py export out - " - "experiments/gpu_network_v2/profile.json" + (" hashes" if hashes_only else "") + "experiments/gpu_network_v2/profile.json" + mode ) log = self.lifecycle.dir / "logs" / f"{role}-network-export-stream.log" try: stream_bounded(ssh_argv(ssh, cmd), archive, log, bound, ssh.timeout) remaining() try: - manifest = verify_export(archive, bound, hashes_only) + manifest = verify_export(archive, bound, hashes_only, dedup) except (tarfile.TarError, KeyError, ValueError) as exc: raise Reject("network_rescue_archive_invalid") from exc # Remote's own manifest digest (stderr) must equal the verified local manifest. @@ -2588,7 +2615,7 @@ def remaining() -> None: cmd = ( f"cd {shlex.quote(root)} && env PYTHONPATH=src {shlex.quote(python)} " "experiments/gpu_network_v2/orchestrate.py export out rescue.tar " - "experiments/gpu_network_v2/profile.json" + (" hashes" if hashes_only else "") + "experiments/gpu_network_v2/profile.json" + mode ) if ssh.run(cmd, "network-export", self.lifecycle.dir / "logs").returncode: raise Reject("network_export_failed") @@ -2598,7 +2625,7 @@ def remaining() -> None: actual = ssh.get(f"{root}/rescue.tar", archive) if actual != expected: raise Reject("network_rescue_sha_mismatch") - manifest = verify_export(archive, bound, hashes_only) + manifest = verify_export(archive, bound, hashes_only, dedup) remaining() if generation != sha256( json.dumps( @@ -2627,6 +2654,7 @@ def remaining() -> None: tar_sha256=actual, files=manifest, hashes_only=hashes_only, + dedup_known=dedup, elapsed_seconds=time.monotonic() - started, ) @@ -2644,11 +2672,89 @@ def hash_rescued(name: str) -> bool: return Path(name).name in HASH_RESCUE_NAMES +def carried_names( + files: dict[str, Any], hashes_only: bool = False, dedup: list[str] | None = None +) -> list[str]: + """Archive members, a pure function of the manifest: all; hash-only small JSON; or dedup, + the first path (sorted, so rank-0 before rank-N) of each sha256 not already held locally.""" + if hashes_only: + return [n for n in files if hash_rescued(n)] + if dedup is None: + return list(files) + seen = set(dedup) + out = [] + for name in sorted(files): + if files[name]["sha256"] not in seen: + seen.add(files[name]["sha256"]) + out.append(name) + return out + + +# Data-parallel ranks write bitwise-identical tensors; only trace.json/summary.json differ. +RANK_IDENTICAL = frozenset({"state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"}) + + +def rank_bytes_differ(files: dict[str, Any]) -> bool: + """Any rank-N state/ef/delta/leaves/checkpoint whose host hash differs from rank-0's.""" + for name, meta in files.items(): + parts = Path(name).parts + rank = next((i for i, p in enumerate(parts) if p.startswith("rank-")), None) + if rank is None or parts[rank] == "rank-0": + continue + rest = parts[rank + 1 :] + if rest and (rest[0] == "checkpoints" or rest == (rest[0],) and rest[0] in RANK_IDENTICAL): + twin = files.get(str(Path(*parts[:rank], "rank-0", *rest))) + if twin is None or twin["sha256"] != meta["sha256"]: + return True + return False + + +def rescued_hashes(tar: Path, prefix: str, scope: str) -> dict[str, str]: + """rel -> sha256 of every rescued file under out/: hashed carried bytes, plus the + manifest hash of a deduplicated file (its content verified once under another path).""" + rescued: dict[str, str] = {} + with tarfile.open(tar) as archive: + names = archive.getnames() + if "manifest.json" in names: + stream = archive.extractfile("manifest.json") + assert stream is not None + for name, meta in json.loads(stream.read()).items(): + if name.startswith(prefix): + rescued[name.removeprefix(prefix)] = meta["sha256"] + carried = set() + for member in archive.getmembers(): + if member.name.startswith("out/" + prefix) and member.isfile(): + rel = member.name.removeprefix("out/" + prefix) + if rel in carried: + raise Reject(scope + "_duplicate_rescue_member") + carried.add(rel) + stream = archive.extractfile(member) + assert stream is not None + digest = sha256(stream.read()) + if rescued.setdefault(rel, digest) != digest: + raise Reject(scope + "_rescue_changed") + return rescued + + +def rescue_mode(profile: dict[str, Any], row: Record) -> tuple[int, bool, list[str] | None]: + """verify_export arguments that re-verify a journaled rescue in its recorded mode.""" + bounds = artifact_budget(profile) + hashes_only, dedup = row.get("hashes_only", False), row.get("dedup_known") + full = not hashes_only and dedup is None # rows recorded before the dedup contract + bound = bounds["full_rescue_tar_bytes_per_host" if full else "rescue_tar_bytes_per_host"] + return bound, hashes_only, dedup + + def export_artifacts( - directory: Path, archive: Path, profile: dict[str, Any], hashes_only: bool = False + directory: Path, + archive: Path, + profile: dict[str, Any], + hashes_only: bool = False, + dedup: list[str] | None = None, ) -> Record: """Archive full paths and every byte (or, hashes_only, every hash plus the small JSON - files above); enforce actual shape-derived byte budget.""" + files above; or dedup, every hash plus each distinct content once); enforce actual + shape-derived byte budget.""" from hypertrain.protocol.hashing import sha256_hex files: dict[str, dict[str, Any]] = {} @@ -2660,10 +2766,15 @@ def export_artifacts( "bytes": path.stat().st_size, "sha256": fsha(path), } - carried = [n for n in files if not hashes_only or hash_rescued(n)] + carried = carried_names(files, hashes_only, dedup) if hashes_only and any(files[n]["bytes"] > HASH_RESCUE_FILE_BYTES for n in carried): raise Reject("export_hash_rescue_file_oversized") bounds = artifact_budget(profile) + tar_bound = bounds[ + "full_rescue_tar_bytes_per_host" + if not hashes_only and dedup is None + else "rescue_tar_bytes_per_host" + ] if sum(v["bytes"] for v in files.values()) > bounds["retained_bytes_per_host"]: raise Reject("actual_artifact_budget_exceeded") metadata = json.dumps(files, sort_keys=True).encode() @@ -2682,7 +2793,7 @@ def export_artifacts( # Nothing archived remotely; the caller re-reads this digest over a separate channel. sys.stdout.buffer.flush() return {"bytes": None, "manifest_sha256": sha256_hex(metadata)} - if archive.stat().st_size > bounds["rescue_tar_bytes_per_host"]: + if archive.stat().st_size > tar_bound: raise Reject("actual_rescue_archive_budget_exceeded") with archive.open("rb") as file: os.fsync(file.fileno()) @@ -2746,9 +2857,12 @@ def place_verified(path: Path, data: bytes, seen: dict[str, Path], dedup: bool) seen[digest] = path -def verify_export(archive: Path, maximum_bytes: int, hashes_only: bool = False) -> Record: +def verify_export( + archive: Path, maximum_bytes: int, hashes_only: bool = False, dedup: list[str] | None = None +) -> Record: """Every carried byte matches the manifest. hashes_only: the manifest still covers every - remote file, and exactly the HASH_RESCUE_NAMES files are carried (none may be withheld).""" + remote file, and exactly the HASH_RESCUE_NAMES files are carried (none may be withheld). + dedup: exactly carried_names() travel, so every manifest hash is carried or held locally.""" import hashlib if archive.stat().st_size > maximum_bytes: @@ -2777,7 +2891,7 @@ def verify_export(archive: Path, maximum_bytes: int, hashes_only: bool = False) for v in files.values() ): raise Reject("rescue_manifest_malformed") - carried = [n for n in files if not hashes_only or hash_rescued(n)] + carried = carried_names(files, hashes_only, dedup) if {m.name for m in entries} != {"manifest.json", *("out/" + n for n in carried)}: raise Reject("rescue_manifest_coverage_mismatch") if any(not n or Path(n).is_absolute() or ".." in Path(n).parts for n in files): @@ -3756,13 +3870,21 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--live", action="store_true") args = parser.parse_args(argv) if args.action == "export": - if args.extra[2:] not in ([], ["hashes"]): - parser.error("export OUT ARCHIVE PROFILE [hashes]") + mode = args.extra[2:3] + known = args.extra[3:] + if ( + mode not in ([], ["hashes"], ["dedup"]) + or known + and mode != ["dedup"] + or not all(len(k) == 64 and set(k) <= set("0123456789abcdef") for k in known) + ): + parser.error("export OUT ARCHIVE PROFILE [hashes | dedup [KNOWN_SHA256 ...]]") exported = export_artifacts( args.input, Path(args.extra[0]), json.loads(Path(args.extra[1]).read_bytes()), - args.extra[2:] == ["hashes"], + mode == ["hashes"], + known if mode == ["dedup"] else None, ) if args.extra[0] == "-": print(exported["manifest_sha256"], file=sys.stderr) diff --git a/scripts/launch_cuda_run.sh b/scripts/launch_cuda_run.sh index 50c3311..cb7281a 100755 --- a/scripts/launch_cuda_run.sh +++ b/scripts/launch_cuda_run.sh @@ -17,7 +17,6 @@ umask 077 TAG=${TAG:-v1} RD=${RD:-/data/role-dispatch} -T=${TOOLS:-$RD/tools} E2E=${E2E:-/root/e2e-deploy-wt/hypertrain-e2e-b/.e2e} OUT=${OUT:-$RD/manifest/out-$TAG} JOB=${JOB:-$RD/manifest/job-$TAG} @@ -29,6 +28,7 @@ PY=${PY:-$RD/runtime/venv/bin/python} MINER_SRC=${MINER_SRC:-/data/pydeps/hypertrain} REPO=${REPO:-$(cd "$(dirname "$0")/.." && pwd)} FIX_BASE=${FIX_BASE:-1a8f91e} +T=${TOOLS:-$REPO/scripts/ops} MINER_SSH=${MINER_SSH:-} MINER_KEY=${MINER_KEY:-} # Signed floor: cutoff >= admitted + staging 1800 s + qualification 9600 s + hard margin 1800 s; @@ -77,7 +77,8 @@ export PYTHONPATH=$HT_TREE/src:$HT_TREE/scripts:$HT_TREE:/data/pydeps:$RD/runtim mkdir -p "$STATE" STAGE=preflight -for f in "$T"/{prepare_record.py,task8-deps.py,rescue_parallel.py,task8-materialize-roots.py} \ +for f in "$T"/{prepare_record.py,task8-deps.py,hf_seed.py,rescue_parallel.py,rescue_dedup.py} \ + "$T"/{task8-materialize-roots.py,prepare_runtime.py} \ "$T"/{backend_bootstrap.py,live_operator.py,auditors.sh} \ "$E2E/beacon/pusher.py" "$E2E/relays/build_relay_configs.py" "$E2E/relays/deploy_relay.sh" \ "$OUT/manifest-envelope.json" "$OUT/manifest.json" "$OUT/profile.json" "$JOB/job.json" \ @@ -120,14 +121,28 @@ case $rc in 0) BACKEND=unregistered ;; 3) BACKEND=cuda ;; 4) die "run ${RID:0:8} BUNDLE=$STATE/bundle rm -rf "$BUNDLE" && mkdir -p "$BUNDLE" && cp -a "$MINER_SRC" "$BUNDLE/hypertrain" find "$BUNDLE" -name __pycache__ -prune -exec rm -rf {} + +# Diff FIX_BASE against the working tree (not HEAD): uncommitted miner changes ship too. if ! grep -q '_retrying("PUT object' "$BUNDLE/hypertrain/miner/core.py"; then - git -C "$REPO" diff "$FIX_BASE" HEAD -- src/hypertrain/miner/core.py src/hypertrain/data/store.py \ + git -C "$REPO" diff "$FIX_BASE" -- src/hypertrain/miner/core.py src/hypertrain/data/store.py \ | patch -p3 --forward --quiet -d "$BUNDLE/hypertrain" || die "transport fixes do not apply onto $MINER_SRC" fi grep -q '_retrying("PUT object' "$BUNDLE/hypertrain/miner/core.py" || die "bundle lacks the PUT retry fix" grep -q 'def _parts' "$BUNDLE/hypertrain/data/store.py" || die "bundle lacks the parallel part fetch" +grep -q '^UPLOAD_WORKERS = ' "$BUNDLE/hypertrain/miner/core.py" || die "bundle lacks the parallel part PUT" +grep -q 'def _local_input' "$BUNDLE/hypertrain/miner/core.py" || die "bundle lacks the theta-only input reuse" +# store.py has no master-side divergence: the bundle copy must be the working tree's bytes. +cmp -s "$REPO/src/hypertrain/data/store.py" "$BUNDLE/hypertrain/data/store.py" \ + || die "bundle data/store.py differs from the working tree" log "PREFLIGHT_OK run=$RID image=$IMAGE job_deadline=$JD backend=$BACKEND miner=${MINER_SSH:-unset}" if [ "$MODE" = --dry-run ]; then log "DRY_RUN_DONE no provider call, nothing spent"; exit 0; fi +# Runtime uid 65532 and the master container (sees /data only) cannot read a /root checkout: +# run a frozen copy of the tools from /data. +if [ "${T#"$RD"/}" = "$T" ]; then + TS=$RD/ops-$TAG + install -d -o 65532 -g 65532 -m 0700 "$TS" + for f in "$T"/*.py "$T"/*.sh; do install -o 65532 -g 65532 -m 0700 "$f" "$TS/"; done + T=$TS +fi if [ ! -e "$STATE/window" ]; then A=$(date +%s); CUT=$((A + WINDOW)); [ "$CUT" -le "$JD" ] || CUT=$JD diff --git a/scripts/ops/auditors.sh b/scripts/ops/auditors.sh new file mode 100755 index 0000000..30dcc38 --- /dev/null +++ b/scripts/ops/auditors.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Two independent contract-2 auditor clients (auditor-0 -> manifest slot 0, auditor-1 -> slot 1). +# CUDA replay runs master-side (store.execute_audit_v2 -> _role_launch "audit" -> owned host). +# Usage: auditors.sh stage | check RUN_ID | once RUN_ID | start RUN_ID | stop | cleanup +set -euo pipefail +IMG=${HT_IMAGE:-ghcr.io/cortexlm/hypertrain@sha256:58e2b42713a9065758796408e7aea4fb3260ccfaa5ad7bad4859fd87320c93d6} +API=${HT_API:-http://cortex-challenge-hypertrain:8000} +NET=${HT_NET:-cortex-challenges} +SRC=${HT_AUDITOR_SEEDS:-/root/.config/cortex-secrets/od-large-e2e} +BASE=/data/role-dispatch/auditors +NAMES=(hypertrain-auditor-0 hypertrain-auditor-1) +cmd=${1:?stage|check|once|start|stop|cleanup}; RID=${2:-} +need_rid() { [[ $RID =~ ^[0-9a-f]{64}$ ]] || { echo "RUN_ID (64 hex) required" >&2; exit 2; }; } +flags() { echo --network "$NET" --read-only --user 65532:65532 --cap-drop ALL --security-opt no-new-privileges \ + --tmpfs /tmp:size=64m -e PYTHONPATH=/data/pydeps -e HOME=/tmp -v /data/pydeps:/data/pydeps:ro \ + -v "$BASE/auditor-$1":/state:ro --entrypoint sh; } +cli() { echo "python -m hypertrain.auditor --api $API --run-id $RID --contract 2 --keyfile /state/auditor.seed"; } +present() { # exact task-owned containers that exist now; docker failure propagates + local all n; all=$(docker ps -a --format '{{.Names}}') || exit 1 + for n in "${NAMES[@]}"; do grep -qxF "$n" <<<"$all" && echo "$n"; done; return 0; } +remove_owned() { + local live; live=$(present) || exit 1 + [[ -z $live ]] || docker rm -f $live || exit 1 + live=$(present) || exit 1 + [[ -z $live ]] || { echo "auditor containers still present: $live" >&2; exit 1; } + echo "auditor containers absent"; } +case $cmd in + stage) install -d -o root -g root -m 0755 $BASE + for i in 0 1; do install -d -o 65532 -g 65532 -m 0700 $BASE/auditor-$i + install -o 65532 -g 65532 -m 0600 $SRC/auditor-$i.seed $BASE/auditor-$i/auditor.seed; done + stat -c '%a %u:%g %n' $BASE/auditor-*/auditor.seed ;; + check) need_rid # GET only: contract 2 and this identity at its exact manifest slot + for i in 0 1; do docker run --rm $(flags $i) "$IMG" -c "python - <<'EOF' +import hypertrain.trainer, httpx +from pathlib import Path +from hypertrain.miner.core import load_keyfile +k=load_keyfile(Path('/state/auditor.seed')); r=httpx.get('$API/v2/runs/$RID',timeout=30); r.raise_for_status(); j=r.json() +aud=j['manifest_envelope']['body']['training']['auditors'] +assert j['contract']==2, j.get('contract') +assert len(aud)>$i and aud[$i]==k.ss58, ('slot $i', k.ss58, aud) +print('READY auditor-$i', k.ss58, 'slot $i') +EOF"; done ;; + once) need_rid # proof: each identity must settle one job with result MATCH; empty queue fails + for i in 0 1; do out=$(docker run --rm $(flags $i) "$IMG" -c "$(cli) --once"); echo "$out" + grep -q '"result": "MATCH"' <<<"$out" || { echo "auditor-$i: no MATCH verdict" >&2; exit 1; } + echo "auditor-$i MATCH"; done ;; + start) need_rid # production loop: re-poll every 20s in-container; a failed poll (transient 5xx while the + # master is busy) is logged with its exit code and retried, never a container restart + for i in 0 1; do docker run -d --name ${NAMES[$i]} --restart always --label io.cortex.task=auditors \ + --log-opt max-size=20m $(flags $i) "$IMG" -c "while :; do $(cli) || echo AUDITOR_POLL_EXIT=\$?; sleep 20; done"; done ;; + stop) remove_owned ;; + cleanup) remove_owned + rm -f $BASE/auditor-0/auditor.seed $BASE/auditor-1/auditor.seed + rmdir $BASE/auditor-0 $BASE/auditor-1 $BASE + test ! -e $BASE && echo "auditors cleanup: seeds+state absent" ;; + *) echo "unknown $cmd" >&2; exit 2 ;; +esac diff --git a/scripts/ops/backend_bootstrap.py b/scripts/ops/backend_bootstrap.py new file mode 100644 index 0000000..7872f36 --- /dev/null +++ b/scripts/ops/backend_bootstrap.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Bootstrap the live CUDA backend from verified two-host qualification, before run POST.""" + +import argparse +import hashlib +import json +import sys +from pathlib import Path + +import hypertrain.trainer # noqa: F401 determinism setup before any torch import +from hypertrain.beacon import parse_round +from hypertrain.challenge.app import Config, load_coord_key +from hypertrain.challenge.store import ChallengeStore +from hypertrain.data.store import LocalFSStore +from hypertrain.gpu_ops import network_qualification as engine +from hypertrain.gpu_ops.journal import fsha +from hypertrain.miner.core import load_keyfile +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages_v2 import RunManifestV2 + +p = argparse.ArgumentParser(description=__doc__) +p.add_argument("--manifest-envelope", type=Path, required=True) +p.add_argument("--owner-seed", type=Path, required=True) +p.add_argument("--config", type=Path, nargs=2, required=True, help="h0 h1: /config.json") +p.add_argument("--result", type=Path, nargs=2, required=True, help="h0 h1: /result.json") +a = p.parse_args() + +raw = a.manifest_envelope.read_bytes() +env = envelope_v2.parse_envelope(raw) +if env.type != "RunManifestV2" or not envelope_v2.verify_envelope(raw): + sys.exit("manifest envelope is not a verified RunManifestV2") +manifest = RunManifestV2.model_validate(env.body) +cfg = Config.from_env() +owner = load_keyfile(a.owner_seed) +if cfg.owner_hotkey is None or owner.ss58 != cfg.owner_hotkey or env.signer != cfg.owner_hotkey: + sys.exit("owner seed / manifest signer is not HYPERTRAIN_OWNER_HOTKEY") + +# The existing verifier runs before any Receipt is sealed or store opened. +for config_path in a.config: + job = engine.check_inputs(engine.Qualification.model_validate_json(config_path.read_bytes())) + if job.manifest != manifest: + sys.exit(f"{config_path}: qualification seed job binds another manifest") +verdict = engine.compare(list(a.result), manifest) +if verdict != {"two_host_bitwise_match": True, "workload_allowed": False}: + sys.exit(f"two-host comparison did not match: {verdict}") + +ref = manifest.training.reference_spec +evidence = { + "run_id": manifest.run_id(), + "backend": "cuda", + "reference_hash": hashlib.sha256(canonicalize(ref.model_dump(mode="json"))).hexdigest(), + "layout_hash": ref.layout.model_dump_json(), + "configs": [fsha(c) for c in a.config], + "results": [fsha(r) for r in a.result], + "summaries": [ + fsha(r.parent / f"round-{w}/published/rank-{k}/summary.json") + for r in a.result + for w in range(2) + for k in range(ref.layout.n_gpus) + ], +} +authority = hashlib.sha256(canonicalize(evidence)).hexdigest() + +store = ChallengeStore( + cfg.state_dir, + cfg.params, + load_coord_key(cfg.coord_key_file), + cfg.owner_hotkey, + parse_round, + LocalFSStore(cfg.state_dir / "objects"), +) +try: + now = store._now(store._db) + receipt = envelope_v2.seal( + owner, + "Receipt", + manifest.run_id(), + {"w": 0, "commit_hash": authority, "received_round": now}, + now + 100, + ) + store._bootstrap_backend_v2(manifest, tuple(a.config), tuple(a.result), canonicalize(receipt)) + print( + json.dumps( + {"run_id": manifest.run_id(), "received_round": now, "authority_hash": authority} + ) + ) +finally: + store.close() diff --git a/scripts/ops/hf_seed.py b/scripts/ops/hf_seed.py new file mode 100644 index 0000000..c0940c0 --- /dev/null +++ b/scripts/ops/hf_seed.py @@ -0,0 +1,104 @@ +"""Rebuild seed objects on a host from the pinned HF genesis checkpoint instead of uploading them. + +A seed object (start_state, ef_in) is a safetensors file whose tensors are either all zero or a +`theta/` copy of the genesis checkpoint's ``. plan() derives that layout from the +controller's local file; the host gets only the header (~100 KB) and a copy list, downloads the +checkpoint with parallel range requests and writes header + zeros + theta. The caller proves the +result with the usual size+sha256 check and falls back to uploading on any mismatch. +""" + +import json +import shlex +import struct +from pathlib import Path + +REPO_ID = "OpenType/neon-1.2-genesis" +# Resolved 2026-10-15 via https://huggingface.co/api/models/OpenType/neon-1.2-genesis +REVISION = "1aa51eb5561719044f206061ba63a0d8e94ab5b0" +URL = f"https://huggingface.co/{REPO_ID}/resolve/{REVISION}/model.safetensors" +SIZE = 1_415_480_056 +SHA256 = "c2b614d2353af47583d4f0604db006e04430a6e84f16066161e748e7fc0b6aa9" +RANGES = 8 +CHUNK = 16 << 20 + + +def plan(local: Path) -> tuple[bytes, list[list]] | None: + """(header bytes, copy list) for `local`, or None when it is not header + zeros + theta.""" + with local.open("rb") as f: + raw = f.read(8) + head = raw + f.read(struct.unpack(" bytes: + spec = json.dumps({"out": out, "src": src, "size": size, "copy": copy}).encode() + return struct.pack(" str: + """bash: fetch the pinned checkpoint to `dst` with `ranges` parallel curls; verify size+sha.""" + q, step = shlex.quote(dst), -(-SIZE // ranges) + gets = "".join( + f"curl -fsSL --retry 3 --connect-timeout 20 -r {i * step}-{min(SIZE, (i + 1) * step) - 1}" + f' -o {q}.r{i} {URL} & p="$p $!"; ' + for i in range(ranges) + ) + parts = " ".join(f"{q}.r{i}" for i in range(ranges)) + return ( + f'set -e; [ "$(sha256sum {q} 2>/dev/null | cut -c1-64)" = {SHA256} ] && exit 0; p=; ' + f"{gets}for x in $p; do wait $x; done; cat {parts} > {q}.part; rm -f {parts}; " + f'[ "$(stat -c %s {q}.part)" = {SIZE} ]; ' + f'[ "$(sha256sum {q}.part | cut -c1-64)" = {SHA256} ]; mv {q}.part {q}' + ) diff --git a/scripts/ops/live_operator.py b/scripts/ops/live_operator.py new file mode 100644 index 0000000..9b2afe3 --- /dev/null +++ b/scripts/ops/live_operator.py @@ -0,0 +1,292 @@ +"""One-shot coordinator/admin commands (open, audits, finalize) on the production master. + +Runs inside the master container. Every value it sends is rebuilt from accepted master state +with the same models, hashes and seals the store validates; nothing here relaxes a guard. +Miner and auditor work stay with their existing CLIs. +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +from contextlib import closing +from pathlib import Path + +import httpx + +import hypertrain.trainer # noqa: F401 +from hypertrain.aggregator.core import load_state +from hypertrain.auditor.replay import AnchorCache, optimizer_hash, pack_state +from hypertrain.data.store import LocalFSStore, put_large +from hypertrain.miner.core import load_keyfile +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.hashing import sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.messages import Finalize, f32hex +from hypertrain.protocol.messages_v2 import ( + PolicyHashes, + RosterEntryV2, + RoundOpenV2, + RunManifestV2, + StartStateV2, +) +from hypertrain.protocol.relay_messages import RelayRegistryV1 +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + + +def index_key(blob: bytes) -> str: + """Key put_large(blob, store.put) returns, computed with a hash-only sink (no writes).""" + return put_large(blob, lambda part: sha256_hex(part)) + + +class Master: + def __init__(self, args: argparse.Namespace) -> None: + if ( + not (args.state_dir / "challenge.db").is_file() + or not (args.state_dir / "objects").is_dir() + ): + raise SystemExit( + "state-dir must contain an existing challenge.db and objects directory" + ) + self.run_id, self.state = args.run_id, args.state_dir + self.objects = LocalFSStore(args.state_dir / "objects") + self.coord = load_keyfile(args.coord_key) + token = args.admin_token_file.read_text().strip() + self.admin = {"Authorization": "Bearer " + token} + self.client = httpx.Client(base_url=args.master, timeout=600) + self.root = f"/v2/runs/{self.run_id}" + + def query(self, sql: str, *params: object) -> sqlite3.Row | None: + uri = (self.state / "challenge.db").as_uri() + "?mode=ro" + with closing(sqlite3.connect(uri, uri=True)) as db: + db.row_factory = sqlite3.Row + return db.execute(sql, params).fetchone() + + def record(self, kind: str, identity: str) -> dict: + row = self.query( + "SELECT data FROM records_v2 WHERE run_id=? AND kind=? AND id=?", + self.run_id, + kind, + identity, + ) + if row is None: + raise SystemExit(f"missing accepted {kind}:{identity}") + return json.loads(row[0]) + + def manifest(self) -> RunManifestV2: + row = self.query("SELECT manifest FROM runs WHERE run_id=?", self.run_id) + assert row is not None + manifest = RunManifestV2.model_validate(envelope_v2.load_json(row[0])) + if manifest.run_id() != self.run_id or manifest.training.coord_pubkey != self.coord.ss58: + raise SystemExit("run or coordinator key differs from the stored manifest") + return manifest + + def call(self, method: str, path: str, body: object = None, admin: bool = False) -> dict: + response = self.client.request( + method, + self.root + path, + content=None if body is None else canonicalize(body), + headers=self.admin if admin else {}, + ) + if response.is_error: + raise SystemExit(f"{method} {path}: {response.status_code} {response.text[:500]}") + return response.json() + + +def refuse_service_run(m: Master) -> None: + """Service-bounded runs need the 4-actor capacity reservation; out of scope here.""" + if m.query( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='service-admission' AND id='run'", + m.run_id, + ): + raise SystemExit("run has a service-admission record; this helper refuses it") + + +def check_open_args(w: int, hotkeys: list[str], offsets: list[int]) -> None: + if w < 0 or not hotkeys or len(set(hotkeys)) != len(hotkeys): + raise SystemExit("need w >= 0 and unique hotkeys") + if ( + len(offsets) != 6 + or offsets[0] < 1 + or any(a >= b for a, b in zip(offsets, offsets[1:], strict=False)) + ): + raise SystemExit("offsets: six strictly increasing positive beacon offsets") + if not 2 <= offsets[5] - offsets[3] <= 200 or offsets[4] <= offsets[2]: + raise SystemExit("offsets: need 2 <= d_final-d_audit <= 200 and d_upload > d_commit") + + +def open_round(m: Master, w: int, hotkeys: list[str], offsets: list[int]) -> dict: + """RoundOpenV2 rebuilt the way open_round_v2 recomputes it (store.py:4892-4939).""" + check_open_args(w, hotkeys, offsets) + refuse_service_run(m) + manifest = m.manifest() + cfg = TrainConfig.from_manifest_v2(manifest) + now = m.call("GET", "")["now_round"] + theta, prev_final = init_params(cfg.model), "0" * 64 + if w: + import torch + + applied = m.record("applied", str(w - 1)) + theta = { + k: torch.from_numpy(v.copy()) + for k, v in load_state(m.objects, applied["out_state"]).theta.items() + } + prev_final = sha256_hex(canonicalize(m.record("finalize", str(w - 1))["body"])) + starts, roster = [], [] + for slot, hot in enumerate(hotkeys): + status = m.call("GET", "/admission/" + hot) + # honored: the challenge assigns live_phase via admission.OWNER_SOLO_HOTKEYS + # (owner-solo override skips trial/screen), so `eligible` alone is authoritative. + if not status["eligible"]: + raise SystemExit("admission is not eligible: " + hot) + cache = AnchorCache() + if w: + a = m.record("anchor", f"{w - 1}:{hot}") + anchor = cache.restore( + Path(a["path"]), + manifest, + a["anchor_hash"], + a["proof_hash"], + a["state_root"], + a["ef_hash"], + expected_hotkey=hot, + expected_round=w - 1, + expected_backend=a["backend"], + ) + else: + anchor = cache.genesis(manifest, hot, theta) + starts.append( + StartStateV2( + run_id=m.run_id, + w=w, + hotkey=hot, + theta_hash=state_hash(theta), + state_object_sha256=index_key(pack_state(theta, anchor.state)), + opt_state_hash=optimizer_hash(anchor.state), + ef_object_sha256=index_key(pack_state(anchor.ef)), + ef_hash=state_hash(anchor.ef), + parent_anchor_hash=anchor.anchor_hash, + global_step0=anchor.state.step, + anchor_verdict_hash=anchor.proof_hash, + ) + ) + rec = status["record"] + roster.append( + { + "hotkey": hot, + "slot": slot, + "q_i": f32hex(1), + "admission_id": rec["admission_id"], + "coldkey_group": rec["coldkey"], + "state": rec["state"], + "eligible_weight": 4194304, + } + ) + current = m.query( + "SELECT data FROM records_v2 WHERE run_id=? AND kind='registry' AND id='current'", + m.run_id, + ) + registry = ( + RelayRegistryV1.model_validate(json.loads(current[0])["body"]) + if current is not None + else RelayRegistryV1.model_validate_json( + m.objects.get(manifest.network.relay_registry_hash) + ) + ) + d = [now + o for o in offsets] + opening = RoundOpenV2( + w=w, + prev_final_hash=prev_final, + theta_hash=state_hash(theta), + outer_state_hash="0" * 64, + center_hash="0" * 64, + roster_hash=sha256_hex(canonicalize(roster)), + honeypot_commit="0" * 64, + d_open=d[0], + d_assign=d[1], + d_commit=d[2], + d_audit=d[3], + d_upload=d[4], + d_final=d[5], + contract_version=2, + policy_hashes=PolicyHashes.model_validate( + {k: getattr(manifest.network, k) for k in PolicyHashes.model_fields} + ), + registry_epoch=registry.epoch, + start_state_index_hash=sha256_hex(canonicalize([s.body() for s in starts])), + audit_mode="anchored-full", + roster=[RosterEntryV2.model_validate(r) for r in roster], + ) + signed = envelope_v2.seal(m.coord, "RoundOpenV2", m.run_id, opening, opening.d_final) + return m.call("POST", "/admin/rounds", signed, True) + + +def included(m: Master, w: int, opening: RoundOpenV2) -> list[str]: + """Same set verified_inputs_v2 yields: roster minus 'excluded' records (store.py:7574-7581).""" + out = [] + for entry in opening.roster: + identity = f"{w}:{entry.hotkey}" + if m.query( + "SELECT 1 FROM records_v2 WHERE run_id=? AND kind='excluded' AND id=?", + m.run_id, + identity, + ): + continue + m.record("commit", identity) # verified_inputs_v2 requires it; refuse before POST + out.append(entry.hotkey) + return sorted(out) + + +def finalize(m: Master, w: int) -> dict: + """One aggregate, then Finalize bound to the applied theta (store.py:7994-7998).""" + refuse_service_run(m) + opening = RoundOpenV2.model_validate(m.record("round", str(w))["body"]) + keep = included(m, w, opening) + applied = m.call("POST", f"/admin/rounds/{w}/aggregate", admin=True) + final = Finalize( + w=w, + final_theta_hash_w1=applied["theta_hash"], + included=keep, + entitlements_root="0" * 64, + ) + signed = envelope_v2.seal(m.coord, "Finalize", m.run_id, final, opening.d_final) + return m.call("POST", f"/admin/rounds/{w}/finalize", signed, True) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--master", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--state-dir", type=Path, required=True) + parser.add_argument("--coord-key", type=Path, required=True) + parser.add_argument("--admin-token-file", type=Path, required=True) + sub = parser.add_subparsers(dest="command", required=True) + o = sub.add_parser("open") + o.add_argument("--w", type=int, required=True) + o.add_argument("--hotkey", action="append", required=True) + o.add_argument("--offsets", required=True, help="d_open..d_final beacon offsets, 6 ints") + sub.add_parser("audits").add_argument("--w", type=int, required=True) + sub.add_parser("finalize").add_argument("--w", type=int, required=True) + args = parser.parse_args(argv) + if args.command == "open": + try: + offsets = [int(x) for x in args.offsets.split(",")] + except ValueError: + raise SystemExit("offsets must be integers") from None + check_open_args(args.w, args.hotkey, offsets) + m = Master(args) + if args.command == "open": + out = open_round(m, args.w, args.hotkey, offsets) + elif args.command == "audits": + out = m.call("POST", f"/admin/rounds/{args.w}/audits", admin=True) + else: + out = finalize(m, args.w) + print(json.dumps(out, sort_keys=True)[:4000]) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/manifest/od_large_manifest.py b/scripts/ops/manifest/od_large_manifest.py new file mode 100644 index 0000000..64efb92 --- /dev/null +++ b/scripts/ops/manifest/od_large_manifest.py @@ -0,0 +1,606 @@ +"""Build the od-large 8x RTX 5090 RunManifestV2 + profile + genesis seed job (DiLoCo H=512). + +Port of .e2e/manifest-prep/od_large_e2e_manifest.py. The dataset is a built shards16 set +(datasets.build, mix od-a-v1); training.dataset is taken from its manifest.json, never packed +here. Miners fetch their assigned units by HTTP Range (HYPERTRAIN_DATA_BASES); only the w=0 seed +job's samples are sliced locally from the shards. + +Run with the snapshot venv, PYTHONPATH=/src:/src. --dry-run writes every +artifact under --out-root (fresh dirs) using throwaway role/owner/escrow keys and never touches +--secrets; the real run takes the approved seeds. Never prints seed material. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import math +import os +import stat +import time +from pathlib import Path + +import numpy as np + +import hypertrain.trainer # noqa: F401 (determinism pins before torch) +from hypertrain.auditor.replay import AnchorCache, pack_state +from hypertrain.datasets.shards16 import ( + SAMPLE_FORMAT_U16, + ShardSet16Manifest, + U16ShardSamples, + shard_name16, +) +from hypertrain.gpu_ops.network_qualification import sources +from hypertrain.ledger.escrow_v2 import genesis_allocation_hash +from hypertrain.protocol import envelope_v2 +from hypertrain.protocol.example import example_manifest +from hypertrain.protocol.hashing import MerkleTree, sha256_hex +from hypertrain.protocol.jcs import canonicalize +from hypertrain.protocol.keys import Keypair +from hypertrain.protocol.messages import f32hex +from hypertrain.protocol.messages_v2 import ( + AdmissionPolicyV2, + AggregationPolicyV2, + ArtifactLimits, + AuditPolicyV2, + DisputePolicyV2, + EconomicsPolicyV2, + IslandJobV1, + RunManifestV2, + audit_lease_rounds, +) +from hypertrain.protocol.relay_messages import RelayRegistryV1 +from hypertrain.trainer.compress import state_hash +from hypertrain.trainer.config import TrainConfig +from hypertrain.trainer.model import init_params + +D, LAYERS, HEADS, HEAD_LAYERS, VOCAB, SEQ = 1024, 24, 16, 4, 50432, 1024 +N_GPUS, GPU_NAME, SM_COUNT = 8, "RTX 5090", 170 +PARAMS = 353861632 +# Inner shape: H=512 (owner asked for 500; 512 * mb1 * ga1 * 8 GPUs = 4096 = 8 units of 512). +H, J, MICRO_BATCH, GRAD_ACCUM = 512, 64, 1, 1 +ASSIGN_UNIT = 512 +STEP_SECONDS = 4 # untraced inner-step bound (est. 1-3 s measured; trace hashing excluded) +EPOCH_SECONDS = 4320 # HYPERTRAIN_EPOCH_SECONDS default (challenge/app.py) +BEACON_PERIOD = 3 +OUTER_LR, OUTER_MOMENTUM, CLIP = 0.7, 0.9, 256.0 +PER_HOST, HOSTS, ROSTER = 7, 2, 4 # 2 qualification rounds + 5 workload executions per host +CLEANUP_MARGIN = 1800 # = schedule hard_margin_seconds +EXP_MARGIN_ROUNDS = 100 # 300 s at the 3 s drand period +ROLES = ("auditor-0", "auditor-1", "referee") +DATA_BASE = "http://185.8.107.43:18980/" + + +def private(path: Path) -> bytes: + mode = path.lstat().st_mode + if not stat.S_ISREG(mode) or mode & 0o077: + raise SystemExit(f"{path.name}: regular 0600 file required") + raw = path.read_bytes() + if len(raw) != 32: + raise SystemExit(f"{path.name}: 32-byte seed required") + return raw + + +def seed_of(secrets: Path, role: str) -> Keypair: + path = secrets / f"{role}.seed" + if not path.exists(): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as f: + f.write(os.urandom(32)) + f.flush() + os.fsync(f.fileno()) + return Keypair(private(path)) + + +def fresh(path: Path) -> Path: + if path.exists(): + raise SystemExit(f"{path}: must not exist (no stale artifacts)") + path.mkdir(parents=True, mode=0o700) + return path + + +def round_seconds() -> int: + """One round's wall time: open to final; d_commit already holds 1.5 * H * STEP_SECONDS.""" + return offsets()[-1] * BEACON_PERIOD + + +def offsets() -> list[int]: + """live_operator d_open..d_final beacon offsets for one H=512 round (3 s rounds). + + open 150 / assign 300 (CPU start-state build, as before); commit after the execution bound + H*STEP_SECONDS (+50% slack); audit +20; upload +600; final = audit + 200 (live_operator caps + d_final - d_audit at 200).""" + commit = 300 + math.ceil(1.5 * H * STEP_SECONDS / BEACON_PERIOD) + return [150, 300, commit, commit + 20, commit + 200, commit + 220] + + +def dataset_spec(manifest_path: Path, holdout_path: Path) -> tuple[dict, ShardSet16Manifest]: + m = ShardSet16Manifest.from_json(manifest_path.read_text()) + hold = ShardSet16Manifest.from_json(holdout_path.read_text()) + if ( + m.sample_format != SAMPLE_FORMAT_U16 + or m.seq_len != SEQ + or m.unit != ASSIGN_UNIT + or m.n_samples % ASSIGN_UNIT + or len(m.unit_sha256s) * m.unit != m.n_samples + or MerkleTree([bytes.fromhex(h) for h in m.unit_sha256s]).root.hex() != m.unit_sha256_root + or MerkleTree([bytes.fromhex(h) for h in m.shard_sha256s]).root.hex() != m.shard_sha256_root + or hold.seq_len != SEQ + ): + raise SystemExit("dataset manifest is not a consistent u16 shards16 set at unit 512") + batch = H * MICRO_BATCH * GRAD_ACCUM * N_GPUS + if ROSTER * batch > m.n_samples: + raise SystemExit("roster x batch exceeds dataset n_samples") + spec = { + "merkle_root": m.merkle_root, + "depth": m.depth, + "n_samples": m.n_samples, + "sample_format": m.sample_format, + "shard_uri_template": m.shard_uri_template, + "shard_sha256_root": m.shard_sha256_root, + "holdout_commit": hold.merkle_root, # public holdout set root (no hidden salt in e2e) + "assign_unit": ASSIGN_UNIT, + "unit_sha256_root": m.unit_sha256_root, + } + return spec, m + + +def relay_registry(staging: Path) -> RelayRegistryV1: + relays = json.loads(staging.read_bytes())["relays"] + if sorted(r["id"] for r in relays) != ["relay-de", "relay-sg", "relay-us"]: + raise SystemExit("exactly the DE/US/SG relay pins required") + return RelayRegistryV1.model_validate( + { + "registry_version": 1, + "epoch": 0, + "previous_registry_hash": None, + "specs": [ + { + "id": r["id"], + "region": r["id"].removeprefix("relay-"), + "https_url": r["https_url"], + "pubkeys": [ + { + "key_id": r["key_id"], + "pubkey": r["pubkey"], + "valid_from_round": 0, + "valid_until_round": 2**40, + } + ], + "max_object_bytes": 2 << 30, + "max_inflight_bytes": 8 << 30, + "codecs": ["ht-sparse-v1", "ht-dense-int8-v1"], + "mode": "transport", + } + for r in sorted(relays, key=lambda r: r["id"]) + ], + } + ) + + +def orchestrate_module(tree: Path): + spec = importlib.util.spec_from_file_location( + "orchestrate", tree / "experiments/gpu_network_v2/orchestrate.py" + ) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def profile_for( + tree: Path, + body: dict, + image: str, + drivers: list[str], + disk: int, + custody: int, + *, + dry_run: bool, +) -> dict: + """Exact od-large/8-GPU profile; qualification compares model/inner/outer/layout bytes.""" + profile = json.loads((tree / "experiments/gpu_network_v2/profile.json").read_bytes()) + profile["model"] = body["model"] + profile["inner"] = body["inner"] + profile["outer"] = body["outer"] + profile["layout"] = body["reference_spec"]["layout"] + execution = round_seconds() # 4632 s: open..final at H=512, 4 s/step + # Qualification = 2 full rounds per host plus probe; staging/rescue windows as before. + probe = 2 * execution + 1200 + rescue_delete = 3600 + qualification = 1800 + probe + rescue_delete + 600 + hard_deadline = 1800 + 600 + rescue_delete + 1800 + PER_HOST * execution + hard_deadline = max(hard_deadline, qualification + 1800) + profile["schedule"] |= { + "hosts": HOSTS, + "executions_per_host": PER_HOST, + "executions_total": HOSTS * PER_HOST, + "logical_identities": 1, + "shadow_participations_per_identity": 0, + "shadow_executions": 0, + "live_executions": HOSTS * (PER_HOST - 2), + "probes": 0, + "fault_trace_reserve": 0, + "staging_seconds": 1800, + "beacon_service_seconds": 600, + "rescue_delete_seconds": rescue_delete, + "execution_transfer_seconds": execution, + "hard_margin_seconds": 1800, + "hard_deadline_seconds": hard_deadline, + } + profile["microqualification"] |= { + "staging_seconds": 1800, + "probe_and_transfer_seconds": probe, + "rescue_delete_seconds": rescue_delete, + "failure_margin_seconds": 600, + "qualification_deadline_seconds": qualification, + "max_full_rounds_per_host": 2, + } + profile["admission"] |= { + "ceiling_usd": "500", + "cleanup_reserve_usd": "5", + "max_host_hourly_usd": "10", + "gpu_name": GPU_NAME, + "sm_count": SM_COUNT, + "max_ancillary_usd": "50", + "gpus_per_host": N_GPUS, + "min_disk_gib": disk >> 30, + } + profile["artifacts"] |= { + "disk_bytes": disk, + "remote_retention": "prune_rescued", + "local_custody_bytes": custody, + "rescue_transfer_seconds": 3300, + "delete_absence_seconds": 300, + } + profile["runtime"] |= {"image_digest": image, "driver_allowlist": drivers, "qualified": False} + orchestrate = orchestrate_module(tree) + bounds = orchestrate.artifact_budget(profile) + orchestrate.execution_quota(profile) + orchestrate.admission_limits(profile) + timing = orchestrate.lifecycle_timing(profile) + if timing["execution_cap"] < execution: + raise SystemExit(f"execution cap {timing['execution_cap']} s < round {execution} s") + # A dry run reports a misfit instead of refusing; the real run refuses. + misfit = [] + if bounds["local_custody_bytes_required"] > custody: + misfit.append("local custody free bytes below prune_rescued requirement") + if bounds["disk_artifacts_peak_bytes"] >= disk: + misfit.append("remote artifact peak exceeds host disk") + if misfit and not dry_run: + raise SystemExit("; ".join(misfit)) + profile["budget_bounds"] = bounds + profile["budget_misfit"] = misfit + return profile + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument( + "--dataset-manifest", + type=Path, + required=True, + help="built shards16 manifest.json (e.g. .../od-a-v1-2m/manifest.json)", + ) + p.add_argument( + "--holdout-manifest", type=Path, help="default: /holdout/manifest.json" + ) + p.add_argument("--tokenizer", type=Path, required=True) + p.add_argument("--tree", type=Path, required=True) + p.add_argument("--registry-manifest", type=Path, required=True) + p.add_argument("--relays", type=Path, required=True) + p.add_argument("--custody-dir", type=Path, required=True) + p.add_argument( + "--out-root", type=Path, required=True, help="fresh out/ and job/ are created beneath it" + ) + p.add_argument( + "--dry-run", + action="store_true", + help="ephemeral keys, no --secrets writes; budget misfit reported not fatal", + ) + for name in ("owner-seed", "escrow-seed", "secrets"): + p.add_argument(f"--{name}", type=Path) + p.add_argument("--coord-pubkey", required=True) + p.add_argument("--hotkey", required=True) + p.add_argument("--driver", action="append", required=True) + p.add_argument("--host-disk-bytes", type=int, required=True) + p.add_argument("--exp-round", type=int) + p.add_argument("--job-deadline", type=int) + a = p.parse_args() + now = int(time.time()) + h_end = now + 43200 + job_deadline = a.job_deadline or now + 4 * 3600 + if job_deadline > h_end - CLEANUP_MARGIN: + raise SystemExit("job deadline exceeds H end minus cleanup margin") + if a.dry_run: + keys = {r: Keypair(os.urandom(32)) for r in ROLES} + owner, escrow = Keypair(os.urandom(32)), Keypair(os.urandom(32)) + else: + if not (a.owner_seed and a.escrow_seed and a.secrets): + raise SystemExit("--owner-seed, --escrow-seed and --secrets required (or --dry-run)") + a.secrets.mkdir(parents=True, exist_ok=True, mode=0o700) + keys = {r: seed_of(a.secrets, r) for r in ROLES} + escrow, owner = Keypair(private(a.escrow_seed)), Keypair(private(a.owner_seed)) + image = "sha256:" + sha256_hex(a.registry_manifest.read_bytes()) + tree_sources = sources(a.tree) + out, job_dir = fresh(a.out_root / "out"), fresh(a.out_root / "job") + vfs = os.statvfs(a.custody_dir) + custody = vfs.f_bavail * vfs.f_frsize + holdout = a.holdout_manifest or a.dataset_manifest.parent / "holdout/manifest.json" + ds, shards = dataset_spec(a.dataset_manifest, holdout) + allocations = [(escrow.ss58, 10000, 1)] + # epochs_per_round: whole HYPERTRAIN_EPOCH_SECONDS epochs per round wall time, >= 1. + epochs = max(1, round(round_seconds() / EPOCH_SECONDS)) + lease = audit_lease_rounds(H, float(STEP_SECONDS)) + policies = { + "admission_policy_hash": AdmissionPolicyV2( + q_base="0000803e", + E=4, + clean_finalizations=12, + work_screen_epoch_rounds=1000, + max_pending=32, + max_trial_replays=2, + max_trials_per_beacon=2, + join_per_beacon=2, + join_burst=4, + ip_prefix_per_beacon=32, + ip_prefix_burst=64, + challenge_rounds=100, + artifact_limits=ArtifactLimits( + max_object_bytes=2 << 30, max_body_bytes=65536, max_chunk_manifest_bytes=1048576 + ), + ), + "economics_policy_hash": EconomicsPolicyV2( + ledger_mode="test", + G_max_units=100, + R_collectible_units=100, + S_min_units=1000, + beta_ppm=1000000, + gammaV_units=0, + s_lower_ppm=1000000, + q_floor=1000000, + genesis_allocation_hash=genesis_allocation_hash(allocations), + reward_authority=a.coord_pubkey, + round_reward_units=epochs * 1000000, + max_total_issuance=20000000 * epochs, + shadow_bootstrap_rounds=12, + ), + "aggregation_policy_hash": AggregationPolicyV2( + arithmetic="flat-cclip-cap-v1", + order="utf8", + center="prev_outer_update", + trust_mode="uniform-verified", + weight_quantum=16777216, + miner_cap_units=4194304, + probation_cap_units=4194304, + owner_group_caps={}, + preclip_norm=f32hex(CLIP), + cclip_tau=f32hex(CLIP), + cclip_iters=1, + ), + "audit_policy_hash": AuditPolicyV2( + lease_rounds=lease, + step_seconds_bound=f32hex(float(STEP_SECONDS)), + max_attempts=2, + max_concurrent_per_auditor=1, + max_running_jobs=2, + max_queued_jobs=32, + max_anchor_age_rounds=1, + max_steps_per_attempt=2 * H, + ), + "dispute_policy_hash": DisputePolicyV2( + referees=[keys["referee"].ss58], + max_open=32, + max_transcript_entries=256, + max_entry_bytes=65536, + fanout=2, + max_referee_reassignments=1, + max_referee_cost_units=100, + ), + "relay_registry_hash": relay_registry(a.relays), + } + body = json.loads(example_manifest().model_dump_json()) + count = VOCAB * D + LAYERS * (12 * D * D + 9 * D + 4 * (D // HEADS)) + 2 * D + assert count == PARAMS + body["model"].update( + arch="od-encoder", + n_layers=LAYERS, + d_model=D, + n_heads=HEADS, + n_kv_heads=HEADS, + d_ff=4 * D, + vocab=VOCAB, + seq_len=SEQ, + param_count=count, + compute_dtype="bf16", + capacity_factor=f32hex(1.0), + aux_loss_coef=f32hex(0.0), + od={ + "preset": "od-large", + "head_layers": HEAD_LAYERS, + "objective": "mlm", + "mask_ratio": f32hex(0.3), + "mask_seed": 7, + "tokenizer_offset": 3, + "warm_start": False, + "record": None, + "decision_rule": "log", + "rps_weight": f32hex(1.0), + "distill_temp": f32hex(1.0), + }, + ) + tok_sha = sha256_hex(a.tokenizer.read_bytes()) + if shards.tokenizer.get("sha256") not in (None, tok_sha): + raise SystemExit("tokenizer bytes differ from the dataset's tokenizer sha256") + body["tokenizer"] = {"name": "answerdotai/ModernBERT-base", "sha256": tok_sha} + body["dataset"] = ds + body["budget"] = {"epochs_per_round": epochs} + body["inner"].update( + H=H, + J=J, + micro_batch=MICRO_BATCH, + grad_accum=GRAD_ACCUM, + state_policy="carry", + rewarmup_steps=0, + ) + body["inner"]["lr_schedule"].update(peak_lr=f32hex(3e-4), warmup=0, stable=100000, decay=0) + body["outer"].update( + opt="nesterov", + lr=f32hex(OUTER_LR), + momentum=f32hex(OUTER_MOMENTUM), + preclip_norm=f32hex(CLIP), + cclip_tau=f32hex(CLIP), + cclip_iters=1, + ) + body["coord_pubkey"] = a.coord_pubkey + body["auditors"] = [keys["auditor-0"].ss58, keys["auditor-1"].ss58] + ref = body["reference_spec"] + ref.update( + image_digest=image, + driver_allowlist=a.driver, + profile="od-bf16-det-eager-v1", + sm_count=SM_COUNT, + ) + ref["layout"] = {"pp": 1, "n_gpus": N_GPUS, "dp_size": N_GPUS, "ep_size": 1, "zero1": False} + profile = profile_for( + a.tree, body, image, a.driver, a.host_disk_bytes, custody, dry_run=a.dry_run + ) + profile_raw = canonicalize(profile) + data_meta = { + "manifest_sha256": sha256_hex(a.dataset_manifest.read_bytes()), + "holdout_manifest_sha256": sha256_hex(holdout.read_bytes()), + "mix_id": shards.extra.get("mix_id"), + "base_urls": [DATA_BASE], + } + ref_doc = canonicalize( + { + "purpose": "od-large stage A MLM DiLoCo H=512 CUDA run", + "dataset": data_meta, + "tokenizer_sha256": tok_sha, + "profile_sha256": sha256_hex(profile_raw), + "sources_sha256": sha256_hex(canonicalize(tree_sources)), + "registry_manifest_sha256": image.removeprefix("sha256:"), + } + ) + ref["spec_doc_sha256"] = sha256_hex(ref_doc) + network = { + **{k: v.digest() for k, v in policies.items()}, + "audit_mode": "anchored-full", + "full_anchor_version": 1, + "capabilities": ["island-replay", "all-level-disputes", "transport-receipts"], + } + draft = RunManifestV2.model_validate( + {"manifest_version": 2, "training": body, "network": network} + ) + theta = init_params(TrainConfig.from_manifest_v2(draft).model) + assert sum(t.numel() for t in theta.values()) == count + body["init_state_hash"] = state_hash(theta) + manifest = RunManifestV2.model_validate( + {"manifest_version": 2, "training": body, "network": network} + ) + policies["audit_policy_hash"].validate_training(manifest.training) + run_id = manifest.run_id() + # Genesis carry seed job (no training step): deterministic init + zero optimizer/EF; its + # samples are the first n rows (units 0..7), proved against the dataset merkle root. + anchor = AnchorCache().genesis(manifest, a.hotkey, theta) + n = manifest.training.batch_samples() + rows = U16ShardSamples( + a.dataset_manifest.parent, shards.n_shards, shards.samples_per_shard, shards.seq_len + ) + if not (a.dataset_manifest.parent / shard_name16(0)).is_file(): + raise SystemExit("seed job needs the local shards next to the dataset manifest") + # ponytail: the full-depth tree over 2^21 rows is built in memory (~3 min, ~200 MB) only to + # emit the n seed proofs; add a cached proof file once round inputs stream per unit. + tree = MerkleTree(rows) + if tree.root.hex() != ds["merkle_root"]: + raise SystemExit("local shard rows differ from the dataset merkle_root") + seed_rows = np.frombuffer(b"".join(rows[i] for i in range(n)), dtype=" h_end_round + EXP_MARGIN_ROUNDS: + raise SystemExit("manifest expiry exceeds H window + margin") + env = envelope_v2.seal(owner, "RunManifestV2", run_id, manifest, exp) + (out / "manifest-envelope.json").write_bytes(canonicalize(env)) + RunManifestV2.model_validate_json((out / "manifest.json").read_bytes()).validate_run_id(run_id) + roles = {r: k.ss58 for r, k in keys.items()} | {"owner": owner.ss58, "escrow-cold": escrow.ss58} + t = manifest.training + index = { + "run_id": run_id, + "dry_run": a.dry_run, + "param_count": count, + "image_digest": image, + "roles": roles, + "hotkey": a.hotkey, + "custody_free_bytes": custody, + "dataset": data_meta, + "summary": { + "inner": {k: getattr(t.inner, k) for k in ("H", "J", "micro_batch", "grad_accum")}, + "batch_samples": n, + "units_per_slot": n // ASSIGN_UNIT, + "outer": t.outer.model_dump(mode="json"), + "dataset": {k: ds[k] for k in ("n_samples", "depth", "assign_unit", "merkle_root")}, + "epochs_per_round": epochs, + "round_seconds": round_seconds(), + "offsets": ",".join(map(str, offsets())), + "audit": { + "lease_rounds": lease, + "absolute_rounds": 2 * lease, + "step_seconds_bound": STEP_SECONDS, + }, + "schedule": { + k: profile["microqualification"][k] + for k in ("probe_and_transfer_seconds", "qualification_deadline_seconds") + } + | { + "hard_deadline_seconds": profile["schedule"]["hard_deadline_seconds"], + "execution_cap": orchestrate_module(a.tree).lifecycle_timing(profile)[ + "execution_cap" + ], + }, + "trace_bytes_per_rank_budget": profile["budget_bounds"]["trace_bytes_per_rank"], + "budget_misfit": profile["budget_misfit"], + }, + "files": { + f"{d.name}/{f.name}": sha256_hex(f.read_bytes()) + for d in (out, job_dir) + for f in sorted(d.iterdir()) + }, + } + (out / "index.json").write_bytes(canonicalize(index)) + print(json.dumps({k: v for k, v in index.items() if k != "files"}, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/scripts/ops/prepare_record.py b/scripts/ops/prepare_record.py new file mode 100644 index 0000000..d3bda57 --- /dev/null +++ b/scripts/ops/prepare_record.py @@ -0,0 +1,293 @@ +"""Build and preflight the signed task-8 reviewed record (adopt | qualification). + +Never launches, rents, or calls a provider. The executor captures fresh read-only provider files +(financial, offers, observations) moments before, passes them with --file, and this script binds +every file by sha256, computes the plan with the controller's own admission_plan, seals the record +with the owner key and preflights it with the same reviewed_launch the lifecycle CLI runs. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import sys +import time +from pathlib import Path +from typing import Any + +# Run-time files the executor may supply; everything else comes from the prepared base record. +FRESH = {"financial", "offers", "observations", "authorization", "config", "profile", "evidence"} +# One real uid247 identity qualifies the same job on both hosts: one bound join/seed context each. +ROLE_FILES = {kind + "_" + role for kind in ("join", "seed_context") for role in ("h0", "h1")} +ADOPT = {"ownership", "provider_journal"} +FRESH_GET = {"financial", "offers", "observations", "authorization"} +EXPIRY_ROUNDS = 10 # receipt is valid for a bounded beacon window only + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def parse_files(items: list[str], action: str) -> dict[str, Path]: + files: dict[str, Path] = {} + for item in items: + name, sep, raw = item.partition("=") + path = Path(raw) + if ( + not sep + or name in files + or name not in FRESH | ROLE_FILES | (ADOPT if action == "adopt" else set()) + or not path.is_absolute() + or path.is_symlink() + or not path.is_file() + ): + raise SystemExit("invalid --file " + item) + files[name] = path + if action == "adopt" and not ADOPT <= set(files): + raise SystemExit("adopt requires --file ownership=... and --file provider_journal=...") + if ROLE_FILES & set(files) and not ROLE_FILES <= set(files): + raise SystemExit("join_h0/h1 and seed_context_h0/h1 are required together") + return files + + +def check_parent(pid: int) -> None: + if pid <= 1 or pid == os.getpid(): + raise SystemExit("--parent-pid must be the long-lived executor shell") + try: + os.close(os.pidfd_open(pid)) + except ProcessLookupError: + raise SystemExit("--parent-pid is not a live process") from None + + +def load_controller(tree: Path) -> Any: + path = (tree / "experiments/gpu_network_v2/orchestrate.py").resolve() + loaded = sys.modules.get("task8_controller") + if loaded is not None: + if Path(loaded.__file__ or "").resolve() != path: + raise SystemExit("loaded controller is not the prepared tree's controller") + return loaded + sys.path[:0] = [str(tree / "src"), str(tree / "scripts")] + spec = importlib.util.spec_from_file_location("task8_controller", path) + if spec is None or spec.loader is None: + raise SystemExit("controller missing in prepared tree") + controller = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = controller + spec.loader.exec_module(controller) + return controller + + +def load_runtime() -> Any: + spec = importlib.util.spec_from_file_location( + "task8_prepare_runtime", Path(__file__).with_name("prepare_runtime.py") + ) + if spec is None or spec.loader is None: + raise SystemExit("prepare_runtime.py missing beside the builder") + runtime = importlib.util.module_from_spec(spec) + spec.loader.exec_module(runtime) + return runtime + + +def warm_seed_genesis(controller: Any, base: dict[str, Any], files: dict[str, Path]) -> None: + """Expensive immutable seed proof before the fresh GET; seed_genesis rehashes every byte.""" + entry, inputs = base["files"], base["role_inputs"] + hotkeys = {json.loads(files["join_" + role].read_bytes())["hotkey"] for role in ("h0", "h1")} + for hotkey in sorted(hotkeys): + controller.seed_genesis( + Path(entry[inputs["job"]]["path"]), + entry[inputs["job"]]["sha256"], + Path(entry["profile"]["path"]), + entry["profile"]["sha256"], + hotkey, + {k: Path(entry[n]["path"]) for k, n in inputs["objects"].items()}, + {k: entry[n]["sha256"] for k, n in inputs["objects"].items()}, + ) + + +def build( + base: dict[str, Any], + files: dict[str, Path], + action: str, + parent_pid: int, + cutoff_unix: int, + admitted_unix: int | None, + owner_seed: Path, + run_id: str, + beacon: int, + output: Path, + provider_key: Path | None = None, +) -> dict[str, Any]: + controller = load_controller(Path(base["tree"])) + from hypertrain.protocol import envelope_v2 + from hypertrain.protocol.hashing import sha256_hex + from hypertrain.protocol.jcs import canonicalize + from hypertrain.protocol.keys import Keypair + + paths = {name: Path(item["path"]) for name, item in base["files"].items()} + paths.update(files) + + def data(name: str) -> Any: + return json.loads(paths[name].read_bytes()) + + profile, cfg, snapshot = data("profile"), data("config"), data("financial") + if "key_file" not in cfg: + # Qualification base: the complete static config plus the two run-time fields. + if provider_key is None or not provider_key.is_absolute() or not provider_key.is_file(): + raise SystemExit("--provider-key-file is required for a qualification base config") + cfg = {**cfg, "key_file": str(provider_key), "cleanup_deadline_unix": cutoff_unix} + config_path = output.with_name(output.stem + "-config.json") + with config_path.open("x") as text: + json.dump(cfg, text, sort_keys=True) + paths["config"] = config_path + horizon = controller.lifecycle_timing(profile)["horizon"] + now = time.time() + if not now < cutoff_unix <= now + horizon: + raise SystemExit("--cutoff-unix must be in the future and inside the reviewed horizon") + if admitted_unix is None and action in ("adopt", "admit"): + raise SystemExit("--admitted-unix is required for adopt and fresh admit") + if admitted_unix is not None and action not in ("adopt", "admit", "qualification"): + raise SystemExit("--admitted-unix is only for adopt and adopted qualification") + # Same subset and planner input as reviewed_launch: foreign rows stay in the signed file. + if len(controller.owned_instances(snapshot, cfg)) != (0 if action == "admit" else 2): + raise SystemExit( + "fresh financial snapshot must hold exactly the two owned instances (none for admit)" + ) + plan = controller.admission_plan( + profile, + data("evidence"), + data("authorization"), + dict(snapshot, instances=[]), + data("offers"), + qualification=True, + priced_seconds=None if admitted_unix is None else cutoff_unix - admitted_unix, + ) + plan_path = output.with_name(output.stem + "-plan.json") + with plan_path.open("x") as text: + json.dump(plan, text) + paths["plan"] = plan_path + record = { + k: v + for k, v in base.items() + if k not in ("preparation_only", "remaining", "admitted_unix", "role_inputs") + } + if ROLE_FILES <= set(files): + inputs = base["role_inputs"] + record["roles"] = { + role: { + "name": role, + "job": inputs["job"], + "objects": inputs["objects"], + "hotkey": json.loads(files["join_" + role].read_bytes())["hotkey"], + "join": "join_" + role, + "seed_context": "seed_context_" + role, + } + for role in ("h0", "h1") + } + elif "roles" not in record: + raise SystemExit("qualification base needs --file join_h0/h1 and seed_context_h0/h1") + owner = Keypair(owner_seed.read_bytes()) + record.update( + action=action, + owner=owner.ss58, + parent_pid=parent_pid, + cutoff_unix=cutoff_unix, + quota=controller.execution_quota(profile), + files={name: {"path": str(p), "sha256": digest(p)} for name, p in sorted(paths.items())}, + ) + if admitted_unix is not None: + record["admitted_unix"] = admitted_unix + receipt = envelope_v2.seal( + owner, + "Receipt", + run_id, + {"w": 0, "commit_hash": sha256_hex(canonicalize(record)), "received_round": beacon}, + beacon + EXPIRY_ROUNDS, + ) + with output.open("xb") as binary: + os.fchmod(binary.fileno(), 0o600) + binary.write(canonicalize({"record": record, "receipt": receipt})) + checked = controller.reviewed_launch(output, owner.ss58, beacon, action) + return { + "review": str(output), + "review_sha256": digest(output), + "plan_sha256": digest(plan_path), + "action": action, + "owner": owner.ss58, + "beacon": beacon, + "parent_pid": parent_pid, + "cutoff_unix": cutoff_unix, + "admitted_unix": admitted_unix, + "adoption": checked["adoption"], + "_checked": checked, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", type=Path, required=True, help="prepared private/review.json") + parser.add_argument("--file", action="append", default=[], help="name=/abs/path") + parser.add_argument("--action", required=True, choices=("admit", "adopt", "qualification")) + parser.add_argument("--parent-pid", type=int, required=True) + parser.add_argument("--cutoff-unix", type=int, required=True) + parser.add_argument("--admitted-unix", type=int) + parser.add_argument("--owner-seed", type=Path, required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--beacon", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--provider-key-file", type=Path) + parser.add_argument("--run-dir", type=Path, help="warm, fresh GET, seal and launch at once") + parser.add_argument("--source-message", help="authorization source for the fresh GET") + parser.add_argument("--live", action="store_true") + args = parser.parse_args() + os.umask(0o077) + if args.beacon < 1: + raise SystemExit("--beacon must be a positive drand round") + check_parent(args.parent_pid) + files = parse_files(args.file, args.action) + base = json.loads(args.base.read_bytes())["record"] + + def seal() -> Any: + result = build( + base, + files, + args.action, + args.parent_pid, + args.cutoff_unix, + args.admitted_unix, + args.owner_seed, + args.run_id, + args.beacon, + args.output.resolve(), + args.provider_key_file, + ) + checked = result.pop("_checked") + print(json.dumps(result, sort_keys=True), flush=True) + return checked + + if args.run_dir is None: + seal() + return + if not args.source_message or args.provider_key_file is None: + raise SystemExit("--run-dir needs --source-message and --provider-key-file") + if FRESH_GET & set(files): + raise SystemExit("--run-dir takes its financial files from its own fresh GET") + controller = load_controller(Path(base["tree"])) + # One invocation: immutable seed proof first, then fresh GET, seal + preflight, launch. + with controller.seed_genesis_scope(): + warm_seed_genesis(controller, base, files) + fresh = load_runtime().financial( + Path(base["tree"]), + Path(base["files"]["config"]["path"]), + args.provider_key_file, + args.source_message, + args.output.resolve().with_name(args.output.stem + "-financial"), + ) + files.update({n: Path(p) for n, p in (item.split("=", 1) for item in fresh["files"])}) + launched = controller.launch_action(args.action, seal(), args.run_dir, live=args.live) + print(json.dumps(launched, sort_keys=True, default=str), flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/ops/prepare_runtime.py b/scripts/ops/prepare_runtime.py new file mode 100644 index 0000000..8be801d --- /dev/null +++ b/scripts/ops/prepare_runtime.py @@ -0,0 +1,440 @@ +"""Task-8 qualification run-time inputs from real material; no keys are generated or printed. + +freeze frozen source snapshot of a tree (same shape the prepare freeze fixture builds) +context one uid247 JoinRequest (hot + approved escrow-cold signature) and its seed_context +financial GET-only provider snapshot, owned-host offers/observations, owner authorization +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import io +import json +import math +import os +import shutil +import sys +import tarfile +import time +import urllib.parse +from decimal import Decimal +from pathlib import Path +from typing import Any + +OPERATION_EXTRA = ( + "scripts/network_gpu_operation.py", + "scripts/network_gpu_seed.py", + "scripts/network_service_proof.py", + "scripts/network_authority_snapshot.py", +) + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write(path: Path, value: Any, mode: int = 0o644) -> Path: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + data = value if isinstance(value, bytes) else json.dumps(value, sort_keys=True).encode() + with path.open("xb") as stream: + os.fchmod(stream.fileno(), mode) + stream.write(data) + return path + + +def freeze(tree: Path, snapshot: Path, image_ref: str) -> dict[str, Any]: + """Copy exactly the qualification + operator sources; record maps and archives by hash.""" + sys.path[:0] = [str(tree / "src")] + from hypertrain.gpu_ops import network_qualification + + canonical = network_qualification.sources(tree) + staged = {rel: digest(tree / rel) for rel in sorted(set(canonical) | set(OPERATION_EXTRA))} + copy = snapshot / "tree" + for rel in staged: + source = tree / rel + if source.is_symlink() or not source.is_file(): + raise SystemExit("source must be a regular file: " + rel) + (copy / rel).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, copy / rel) + if network_qualification.sources(copy) != canonical: + raise SystemExit("frozen copy differs from the tree map") + maps = { + "qualification-sources.json": canonical, + "operation-sources.json": { + **canonical, + "scripts/network_gpu_operation.py": staged["scripts/network_gpu_operation.py"], + }, + "sources.json": staged, + "files.json": { + rel: { + "origin": str(copy / rel), + "size": (copy / rel).stat().st_size, + "sha256": d, + } + for rel, d in staged.items() + }, + } + for name, values in maps.items(): + write(snapshot / name, values) + write(snapshot / "qualification-source.tar", network_qualification.bundle(copy)) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w") as archive: + for rel in staged: + data = (copy / rel).read_bytes() + member = tarfile.TarInfo(rel) + member.size, member.mode, member.mtime = len(data), 0o600, 0 + archive.addfile(member, io.BytesIO(data)) + write(snapshot / "effective-source.tar", buffer.getvalue()) + + def artifact(path: Path) -> dict[str, Any]: + return {"path": str(path), "bytes": path.stat().st_size, "sha256": digest(path)} + + record = { + "snapshot": str(snapshot), + "archive": artifact(snapshot / "effective-source.tar"), + "qualification_archive": artifact(snapshot / "qualification-source.tar"), + "maps": {name: artifact(snapshot / name) for name in maps}, + "approved_pins": { + rel: staged[rel] + for rel in ( + "scripts/network_service_proof.py", + "scripts/network_gpu_operation.py", + "experiments/gpu_network_v2/orchestrate.py", + ) + }, + "service_source_observed": {}, + "qualification_parser": {}, + "image_candidate": {"ref": image_ref}, + "paid_GO": False, + } + write(snapshot / "freeze.json", record) + return { + "freeze": str(snapshot / "freeze.json"), + "sha256": digest(snapshot / "freeze.json"), + } + + +def seed(path: Path) -> bytes: + """Raw 32-byte or 64-hex-char 0600 seed; contents never leave this process.""" + if path.is_symlink() or not path.is_file() or path.stat().st_mode & 0o077: + raise SystemExit(f"{path}: regular 0600 seed file required") + raw = path.read_bytes() + if len(raw) == 32: + return raw + text = raw.strip() + if len(text) == 64: + return bytes.fromhex(text.decode("ascii")) + raise SystemExit(f"{path}: 32-byte or 64-hex seed required") + + +def context( + tree: Path, + out: Path, + job_path: Path, + hotkey_seed: Path, + cold_seed: Path, + owner: str, + expires_beacon: int, + destination: Path, +) -> dict[str, Any]: + sys.path[:0] = [str(tree / "src")] + from hypertrain.miner.admission import sign_join + from hypertrain.protocol.envelope_v2 import verify_join + from hypertrain.protocol.keys import Keypair + from hypertrain.protocol.messages_v2 import HardwareHint, IslandJobV1 + + job = IslandJobV1.model_validate_json(job_path.read_bytes()) + profile = json.loads((out / "profile.json").read_bytes()) + hot, cold = Keypair(seed(hotkey_seed)), Keypair(seed(cold_seed)) + if hot.ss58 != json.loads((out / "index.json").read_bytes())["hotkey"]: + raise SystemExit("hotkey seed is not the manifest index hotkey") + request_id = hashlib.sha256(f"task8-qualification|{job.run_id}|{hot.ss58}".encode()).hexdigest() + join = sign_join( + hot, + cold, + run_id=job.run_id, + request_id=request_id, + expires_beacon=expires_beacon, + policy_hash=job.manifest.network.admission_policy_hash, + hardware_hint=HardwareHint( + device_name=profile["admission"]["gpu_name"], + device_count=profile["layout"]["n_gpus"], + driver=profile["runtime"]["driver_allowlist"][0], + ), + ) + if not verify_join(join, job.run_id, 1): + raise SystemExit("join does not verify") + join_path = write(destination / "join.json", join.model_dump(mode="json")) + admission_id = hashlib.sha256( + f"join|{job.run_id}|{join.coldkey}|{join.request_id}".encode() + ).hexdigest() + seed_context = { + "run_id": job.run_id, + "owner": owner, + "hotkey": join.hotkey, + "coldkey": join.coldkey, + "request_id": join.request_id, + "policy_hash": join.policy_hash, + "job_sha256": digest(job_path), + "admission_id": admission_id, + "state": "PROBATION", + "join_sha256": digest(join_path), + } + context_path = write(destination / "seed_context.json", seed_context) + return { + "join": str(join_path), + "seed_context": str(context_path), + "hotkey": join.hotkey, + "coldkey": join.coldkey, + "admission_id": admission_id, + # Same identity qualifies the same job on both hosts: pass these for _h0 and _h1. + "builder_files": [ + f"{kind}_{role}={path}" + for kind, path in (("join", join_path), ("seed_context", context_path)) + for role in ("h0", "h1") + ], + } + + +def financial( + tree: Path, + config: Path, + key_file: Path, + source_message: str, + destination: Path, +) -> dict[str, Any]: + """GET-only. Offers are the two owned instances' live rows (adopt: no market offers).""" + sys.path[:0] = [str(tree / "src")] + spec = importlib.util.spec_from_file_location( + "runtime_controller", tree / "experiments/gpu_network_v2/orchestrate.py" + ) + assert spec is not None and spec.loader is not None + controller = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = controller + spec.loader.exec_module(controller) + from hypertrain.gpu_ops.journal import Journal + from hypertrain.gpu_ops.provider import Provider + + cfg = json.loads(config.read_bytes()) + profile = json.loads(Path(cfg["network_profile_file"]).read_bytes()) + run = destination / "provider-run" + run.mkdir(parents=True, mode=0o700) + provider = Provider( + cfg["base_url"], + key_file.read_text().strip(), + run, + Journal(run), + False, + ) + requested = int(time.time()) + snapshot = controller.refresh_readonly(provider) + account = provider.call("GET", "/api/v0/users/current/", "runtime-account") + received = int(time.time()) + owned = controller.owned_instances(snapshot, cfg) + if len(owned) not in (0, 2) or not account.ok(): + raise SystemExit("fresh inventory must hold the two owned instances or none (admit)") + rows = {row["machine_id"]: row for row in snapshot["instances"]} + if not owned: + # Fresh admit: live asks on the signed config machines, GET only. + gpus = controller.admission_limits(profile)["gpus"] + for h in cfg["hosts"]: + q = { + "machine_id": {"eq": h["machine_id"]}, + "num_gpus": {"eq": gpus}, + "rentable": {"eq": True}, + "rented": {"eq": False}, + } + r = provider.call( + "GET", + "/api/v0/bundles/?" + urllib.parse.urlencode({"q": json.dumps(q)}), + "runtime-offer", + ) + asks = r.parsed.get("offers") if r.ok() else None + if not isinstance(asks, list) or not asks: + raise SystemExit("no fresh ask for machine " + str(h["machine_id"])) + rows[h["machine_id"]] = min(asks, key=lambda o: (Decimal(str(o["dph_total"])), o["id"])) + # Mission floor re-checked on the live ask that create will use, not the base snapshot. + for link in ("inet_up", "inet_down"): + v = rows[h["machine_id"]].get(link) + if not ( + isinstance(v, (int, float)) + and not isinstance(v, bool) + and math.isfinite(v) + and v >= 800 + ): + raise SystemExit(f"fresh ask {link} below 800 Mbit/s for {h['machine_id']}") + received = int(time.time()) + hours = Decimal(controller.lifecycle_timing(profile)["horizon"]) / 3600 + # Foreign account instances for a full horizon; the owned pair is priced by the plan's offers. + # A stopped instance bills storage only; anything else (running, loading, unknown) bills dph. + mine = {h["machine_id"] for h in cfg["hosts"]} + liabilities = ( + sum( + Decimal( + str( + r["storage_total_cost"] + if r.get("actual_status") == "exited" and r.get("cur_state") == "stopped" + else r["dph_total"] + ) + ) + for r in snapshot["instances"] + if r["machine_id"] not in mine + ) + * hours + ) + # Spend so far on the adopted pair: provider charge rows, or run time x rate if larger + # (daily charge rows lag). Upper bound, never an invented zero. + sources = {"instance-" + str(row["id"]) for row in owned} + # Fresh admit has no owned rows yet: every instance charge in the window counts. + charged = sum( + Decimal(str(c["amount"])) + for c in snapshot["charges"] + if c.get("source") in sources or (not owned and c.get("type") == "instance") + ) + accrued = sum( + Decimal(str(row["dph_total"])) * Decimal(received - row["start_date"]) / 3600 + for row in owned + ) + prior_attempts_usd = f"{max(charged, accrued):.6f}" + verified = int(time.time()) + stamp = { + "requested_unix": requested, + "received_unix": received, + "verified_unix": verified, + } + snapshot.update( + stamp, + account_id=account.parsed["id"], + uncapped_charges=False, + prior_attempts_usd=prior_attempts_usd, + active_liabilities_usd=f"{liabilities:.6f}", + prior_attempts_basis={ + "charged_usd": str(charged), + "accrued_usd": f"{accrued:.6f}", + }, + ) + keys = ( + "id", + "machine_id", + "num_gpus", + "gpu_name", + "dph_total", + "storage_cost", + "inet_up_cost", + "inet_down_cost", + "disk_space", + "cpu_cores_effective", + "cpu_ram", + "driver_version", + ) + offers = [{**{k: rows[h["machine_id"]].get(k) for k in keys}, **stamp} for h in cfg["hosts"]] + observations = [ + { + "machine_id": h["machine_id"], + "actual_status": rows[h["machine_id"]].get("actual_status"), + **stamp, + } + for h in cfg["hosts"] + ] + limits = controller.admission_limits(profile) + authorization = { + "experiment": "hypertrain-network-v2", + "currency": "USD", + "ceiling_usd": limits["ceiling_usd"], + "max_instances": 2, + "max_seconds_per_instance": controller.lifecycle_timing(profile)["horizon"], + "source_message": source_message, + "date": time.strftime("%Y-%m-%d", time.gmtime(verified)), + } + names = { + "financial": snapshot, + "offers": offers, + "observations": observations, + "authorization": authorization, + } + paths = { + name: write(destination / (name + ".json"), value, 0o600) for name, value in names.items() + } + return { + "files": [f"{n}={p}" for n, p in paths.items()], + "verified_unix": verified, + "must_seal_before_unix": verified + 30, + } + + +def evidence(base: Path, destination: Path) -> dict[str, Any]: + """Pre-launch qualification evidence from root-reviewed hashes; nothing is measured here.""" + record = json.loads(base.read_bytes())["record"] + files = {name: Path(item["path"]) for name, item in record["files"].items()} + if any(digest(files[name]) != item["sha256"] for name, item in record["files"].items()): + raise SystemExit("base file changed since preparation") + profile = json.loads(files["profile"].read_bytes()) + if "sha256:" + digest(files["registry"]) != profile["runtime"]["image_digest"]: + raise SystemExit("registry index is not the pinned image digest") + value = { + # Lifecycle/supervisor sources are the task-26/28 root-gated hashes bound in the base. + "root_verified_bounded_lifecycle": True, + "registry_image_verified": True, + "candidate_image_digest": profile["runtime"]["image_digest"], + "registry_index_sha256": digest(files["registry"]), + "registry_platform_sha256": digest(files["registry_platform"]), + "staged_contract_sha256": digest(files["sources"]), + "staged_bundle_sha256": digest(files["bundle"]), + "deadline_supervisor_contract_sha256": digest(files["lifecycle_hashes"]), + "measured_runtime": None, + } + path = write(destination / "evidence.json", value) + return {"file": f"evidence={path}", "sha256": digest(path)} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="step", required=True) + f = sub.add_parser("freeze") + f.add_argument("--tree", type=Path, required=True) + f.add_argument("--snapshot", type=Path, required=True) + f.add_argument("--image-ref", required=True) + c = sub.add_parser("context") + for name in ("tree", "out", "job", "hotkey-seed", "cold-seed", "destination"): + c.add_argument("--" + name, type=Path, required=True) + c.add_argument("--owner", required=True) + c.add_argument("--expires-beacon", type=int, required=True) + m = sub.add_parser("financial") + for name in ("tree", "config", "key-file", "destination"): + m.add_argument("--" + name, type=Path, required=True) + m.add_argument("--source-message", required=True) + v = sub.add_parser("evidence") + v.add_argument("--base", type=Path, required=True) + v.add_argument("--destination", type=Path, required=True) + a = parser.parse_args() + os.umask(0o077) + if a.step == "evidence": + result = evidence(a.base.resolve(), a.destination.resolve()) + elif a.step == "freeze": + result = freeze(a.tree.resolve(), a.snapshot.resolve(), a.image_ref) + elif a.step == "context": + result = context( + a.tree.resolve(), + a.out.resolve(), + a.job.resolve(), + a.hotkey_seed, + a.cold_seed, + a.owner, + a.expires_beacon, + a.destination.resolve(), + ) + else: + result = financial( + a.tree.resolve(), + a.config.resolve(), + a.key_file, + a.source_message, + a.destination.resolve(), + ) + print(json.dumps(result, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/ops/probe_and_launch.py b/scripts/ops/probe_and_launch.py new file mode 100755 index 0000000..d8d268b --- /dev/null +++ b/scripts/ops/probe_and_launch.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +# ruff: noqa: E501, S310, S603, S607 (verbatim controller tool: fixed Vast API URL, argv lists) +"""Find two healthy Vast 8x RTX 5090 hosts (rent, wait for boot, delete), then prepare and launch. + +Vast hosts often fail container start with "failed to inject CDI devices" (GPUs not attachable); +the launcher's boot() then kills the whole admission. Each candidate is rented with the exact +lifecycle image, watched until running+ssh or failure, and always deleted (absence verified) +before the real admission rents it again. A booted probe must also prove real bandwidth (16-stream +ssh egress to this controller, ranged HF ingress): advertised inet_up does not predict it. +Log lines: PROBE, PAIR, LAUNCHED, GIVEUP. +ponytail: an ask can vanish between probe and admission; the launcher then fails at admit and +this script must be re-run. +""" + +import concurrent.futures as cf +import json +import os +import subprocess +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.request + +from hf_seed import URL as HF_URL + +D = "/data/role-dispatch" +API = "https://console.vast.ai/api/v0" +KEY = open(f"{D}/keys/vast.key").read().strip() +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +LAUNCHER = f"{REPO}/scripts/launch_cuda_run.sh" +HT_TREE = os.environ.get("HT_TREE") or sys.exit("HT_TREE must name the controller tree") +SRC = os.environ.get("SRC", "v1") +OUT, JOB, SRC_PREP = f"{D}/manifest/out-{SRC}", f"{D}/manifest/job-{SRC}", f"{D}/prep-{SRC}" +# Frozen controller tree (prep-v3 = direct-ssh launcher); context stays the signed run's. +FREEZE_PREP = os.environ.get("FREEZE_PREP", SRC_PREP) +PROFILE = json.load(open(f"{OUT}/profile.json")) +ALLOW = set(PROFILE["runtime"]["driver_allowlist"]) +ADM = PROFILE["admission"] +IMAGE = json.load(open(f"{D}/prep-v1/base/private/lifecycle-config.json"))["image"] +OFFER_KEYS = ( + "cpu_cores_effective", + "cpu_ram", + "disk_space", + "dph_total", + "driver_version", + "gpu_name", + "id", + "inet_down", + "inet_down_cost", + "inet_up", + "inet_up_cost", + "machine_id", + "num_gpus", + "reliability2", + "storage_cost", +) +MIN_MBPS = 800 # network_gpu_prepare.py MIN_INET_MBPS +BOOT_S, CYCLE_S, MAX_CYCLES = 300, 120, int(os.environ.get("MAX_CYCLES", "24")) +# Measured gates: 16 x 256 MiB ssh streams host->controller for <=30 s, one ranged HF curl. +EGRESS_MIN, INGRESS_MIN, BW_S, STREAMS = 700, 400, 30, 16 +KEYDIR = tempfile.mkdtemp(prefix="ht-probe-") +PROBE_KEY = f"{KEYDIR}/id_ed25519" + + +def us_first(o): + return (not str(o.get("geolocation") or "").endswith(", US"), o["dph_total"]) + + +def log(*a): + print(time.strftime("%FT%TZ", time.gmtime()), *a, flush=True) + + +def call(method, path, body=None, base=API): + for attempt in range(8): + req = urllib.request.Request( + base + path, + method=method, + data=None if body is None else json.dumps(body).encode(), + headers={"Authorization": "Bearer " + KEY, "Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + if e.code != 429 and e.code < 500 or attempt == 7: + raise + time.sleep(min(60, 3 * 2**attempt)) + + +def owned_machines(): + rows = call("GET", "/instances/?owner=me", base="https://console.vast.ai/api/v1") + return {i["machine_id"] for i in rows.get("instances", [])} + + +def offers(machines=None): + q = { + "gpu_name": {"eq": "RTX 5090"}, + "num_gpus": {"eq": 8}, + "rentable": {"eq": True}, + "disk_space": {"gte": ADM["min_disk_gib"]}, + "direct_port_count": {"gte": 1}, + "type": "on-demand", + "limit": 200, + "order": [["dph_total", "asc"]], + } + if machines: + q["machine_id"] = {"in": sorted(machines)} + return [ + o + for o in call("POST", "/bundles/", q)["offers"] + if o["driver_version"] in ALLOW + and o["dph_total"] <= float(ADM["max_host_hourly_usd"]) + and o["cpu_cores_effective"] >= ADM["min_cpu_cores"] + and o["cpu_ram"] / 1024 >= ADM["min_host_ram_gib"] + and o["inet_up"] >= MIN_MBPS + and o["inet_down"] >= MIN_MBPS + ] + + +def bandwidth(ip, port): + """(egress, ingress) Mbit/s measured on a running probe host, or None when ssh never answers.""" + ssh = [ + "ssh", + "-i", + PROBE_KEY, + "-o", + "IdentitiesOnly=yes", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=no", + "-o", + "UserKnownHostsFile=/dev/null", + "-o", + "ConnectTimeout=15", + "-o", + "Compression=no", + "-p", + str(port), + f"root@{ip}", + ] + t0 = time.time() + while subprocess.run([*ssh, "true"], capture_output=True, timeout=60).returncode: + if time.time() - t0 > 120: # key propagation + sshd start + return None + time.sleep(10) + got = [0] * STREAMS + + def stream(i): + p = subprocess.Popen( + [*ssh, f"head -c {256 << 20} /dev/zero"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + end = time.time() + BW_S + try: + while time.time() < end and (b := p.stdout.read1(1 << 20)): + got[i] += len(b) + finally: + p.kill() + p.wait() + + t0 = time.time() + threads = [threading.Thread(target=stream, args=(i,)) for i in range(STREAMS)] + for t in threads: + t.start() + for t in threads: + t.join() + egress = sum(got) * 8 / (time.time() - t0) / 1e6 + r = subprocess.run( + [ + *ssh, + f"curl -s -o /dev/null -r 0-{512 << 20} --max-time {BW_S} " + f"-w '%{{size_download}} %{{time_total}}' {HF_URL}", + ], + capture_output=True, + text=True, + timeout=BW_S + 60, + ) + try: + size, secs = map(float, r.stdout.split()[-2:]) + ingress = size * 8 / secs / 1e6 + except ValueError: + ingress = 0.0 + return egress, ingress + + +def probe(o): + mid = o["machine_id"] + try: + put = call( + "PUT", + f"/asks/{o['id']}/", + { + "image": IMAGE, + "disk": ADM["min_disk_gib"], + "runtype": "ssh_direc ssh_proxy", + "target_state": "running", + "cancel_unavail": True, + "onstart": "true", + "label": f"ht-probe-{mid}", + }, + ) + except Exception as e: # ask vanished / HTTP error: not healthy, nothing rented + return mid, f"rent_failed {e}" + iid = put.get("new_contract") + if not iid: + return mid, f"rent_failed {str(put)[:120]}" + verdict, t0 = "boot_timeout", time.time() + try: + while time.time() - t0 < BOOT_S: + st = call("GET", f"/instances/{iid}/?owner=me").get("instances") or {} + msg = st.get("status_msg") or "" + direct = ((st.get("ports") or {}).get("22/tcp") or [{}])[0].get("HostPort") + if st.get("actual_status") == "running" and direct: + up = f"{int(time.time() - t0)}s direct={st.get('public_ipaddr')}:{direct}" + call( + "POST", + f"/instances/{iid}/ssh/", + {"ssh_key": open(PROBE_KEY + ".pub").read().strip()}, + ) + try: + bw = bandwidth(st.get("public_ipaddr"), direct) + except subprocess.TimeoutExpired: + bw = None + if bw is None: + verdict = f"slow ssh_unreachable {up}" + else: + ok = bw[0] >= EGRESS_MIN and bw[1] >= INGRESS_MIN + verdict = ( + f"{'HEALTHY' if ok else 'slow'} {up} egress={bw[0]:.0f}Mbit/s " + f"ingress={bw[1]:.0f}Mbit/s" + ) + break + if ( + "CDI" in msg + or "Error response" in msg + or st.get("actual_status") in ("exited", "offline") + or st.get("cur_state") == "stopped" + ): + verdict = "failed " + msg.strip().splitlines()[-1][:120] if msg else "failed" + break + time.sleep(10) + finally: + for _ in range(6): + try: + call("DELETE", f"/instances/{iid}/") + if not call("GET", f"/instances/{iid}/?owner=me").get("instances"): + break + except Exception as e: + log("PROBE delete retry", mid, iid, e) + time.sleep(10) + else: + log("PROBE DELETE_UNCONFIRMED", mid, iid) + return mid, verdict + + +def run(cmd, **kw): + return subprocess.run(["bash", "-c", cmd], capture_output=True, text=True, **kw) + + +def launch(pair, tag): + prep, now = f"{D}/prep-{tag}", int(time.time()) + for _ in range(24): # a just-deleted probe takes a while to be listed as rentable again + fresh = {o["machine_id"]: o for o in offers(pair)} + if set(fresh) == set(pair): + break + time.sleep(15) + if set(fresh) != set(pair): + return f"asks gone {sorted(set(pair) - set(fresh))}" + sel = [{**{k: fresh[m][k] for k in OFFER_KEYS}, "received_unix": now} for m in pair] + sel_path = f"{D}/prep-r5/selected-pair-{tag}-offers.json" + with open(sel_path, "w") as f: + json.dump(sel, f) + os.chown(sel_path, 65532, 65532) + os.chmod(sel_path, 0o600) + env = ( + f"export PYTHONHOME={D}/runtime/local CUDA_VISIBLE_DEVICES= HOME={D} " + f"PYTHONPATH={HT_TREE}/src:{HT_TREE}/scripts:" + f"{HT_TREE}:/data/pydeps:{D}/runtime/venv/lib/python3.12/site-packages;" + ) + as_rt = "setpriv --reuid=65532 --regid=65532 --clear-groups" + py = f"{D}/runtime/venv/bin/python" + r = run(f"""set -e; {env} +install -m 0755 {REPO}/scripts/network_gpu_prepare.py /tmp/ngp-vast.py +install -m 0755 {REPO}/scripts/ops/prepare_runtime.py /tmp/prepare-runtime-vast.py +install -d -o 65532 -g 65532 -m 0700 {prep}; cp -a {FREEZE_PREP}/freeze {SRC_PREP}/context {prep}/; cd {D} +{as_rt} {py} /tmp/ngp-vast.py --freeze {prep}/freeze/freeze.json --output {prep}/base \ + --cpu-genesis-unix {now} --qualification-base {OUT} {JOB} {sel_path} \ + {D}/custody/index.manifest.json {D}/custody/amd64.manifest.json > /dev/null +{as_rt} {py} /tmp/prepare-runtime-vast.py evidence --base {prep}/base/private/review-base.json \ + --destination {prep}/evidence > /dev/null +HT_TREE= TAG={tag} PREP={prep} RUN={D}/run-{tag} OUT={OUT} JOB={JOB} \ + timeout 250 {LAUNCHER} --dry-run""") + if "DRY_RUN_DONE" not in r.stdout: + return f"prepare/dry-run failed: {(r.stdout + r.stderr)[-600:]}" + subprocess.Popen( + [ + "setsid", + "bash", + "-c", + f"HT_TREE= TAG={tag} PREP={prep} RUN={D}/run-{tag} OUT={OUT} JOB={JOB} " + f"exec {LAUNCHER} --yes-spend > {D}/launch-{tag}.log 2>&1 < /dev/null", + ], + start_new_session=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return None + + +def main(): + tag = sys.argv[1] + bad: dict[int, int] = {} + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", PROBE_KEY], check=True) + for cycle in range(MAX_CYCLES): + busy = owned_machines() + cands = [ + o for o in offers() if o["machine_id"] not in busy and bad.get(o["machine_id"], 0) < 2 + ] + log( + "CYCLE", + cycle, + "candidates", + [(o["machine_id"], o["driver_version"], round(o["dph_total"], 2)) for o in cands], + ) + # US hosts first (controller link), then price; one wave, 2 s apart (under Vast's 429 rate). + cands.sort(key=us_first) + with cf.ThreadPoolExecutor(len(cands[:10]) or 1) as ex: + futures = [] + for o in cands[:10]: + futures.append(ex.submit(probe, o)) + time.sleep(2) + results = [f.result() for f in futures] + healthy = [] + for mid, verdict in results: + log("PROBE", mid, verdict) + if verdict.startswith("HEALTHY"): + healthy.append(mid) + elif verdict.startswith("failed"): + bad[mid] = bad.get(mid, 0) + 1 + if len(healthy) >= 2: + rank = {o["machine_id"]: us_first(o) for o in cands} + pair = sorted(healthy, key=rank.get)[:2] + log("PAIR", pair) + err = launch(pair, tag) + if err is None: + log("LAUNCHED", tag, f"log={D}/launch-{tag}.log") + return 0 + log("LAUNCH_SKIPPED", err) + time.sleep(CYCLE_S) + log("GIVEUP no healthy compliant 8x5090 pair after", MAX_CYCLES, "cycles") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ops/rescue_dedup.py b/scripts/ops/rescue_dedup.py new file mode 100644 index 0000000..8b93ccb --- /dev/null +++ b/scripts/ops/rescue_dedup.py @@ -0,0 +1,290 @@ +"""Native network-v2 rescue with deduplicated transport (r14 recovery adapter, outside the 118 map). + +Runs the unchanged NetworkRuntime.rescue(role, fresh=True): same generation, same pinned SSH, +same bound/deadline, same verify_export, same remote-digest check and the same native +network_rescued journal row. Only stream_bounded is replaced in this process: the host sends +each distinct file content once and the exact export tar (out/ + manifest.json) is +rebuilt locally from hash-verified blobs. Read-only on the host; the producer is sent on stdin. +""" + +# ruff: noqa: E501 (verbatim controller tool) +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import re +import shlex +import shutil +import signal +import subprocess +import sys +import tarfile +import threading +import time +from collections.abc import Callable +from pathlib import Path, PurePosixPath + +HEX = re.compile(r"[0-9a-f]{64}") +MANIFEST_MAX = 1048576 # verify_export's own manifest bound +MAX_FILES = 100000 + +PRODUCER = r""" +import hashlib, io, json, os, sys, tarfile +files, sent = {}, set() +with tarfile.open(fileobj=sys.stdout.buffer, mode="w|", format=tarfile.PAX_FORMAT) as out: + for dirpath, dirs, names in os.walk("out"): + dirs.sort() + for d in dirs: + if os.path.islink(os.path.join(dirpath, d)): + sys.exit("export_symlink_rejected " + os.path.join(dirpath, d)) + for name in sorted(names): + path = os.path.join(dirpath, name) + if os.path.islink(path) or not os.path.isfile(path): + sys.exit("export_symlink_rejected " + path) + digest = hashlib.sha256() + with open(path, "rb") as f: + while block := f.read(1 << 22): + digest.update(block) + size = f.tell() + sha = digest.hexdigest() + files[os.path.relpath(path, "out")] = {"bytes": size, "sha256": sha} + if sha in sent: + continue + with open(path, "rb") as f: + info = tarfile.TarInfo("blobs/" + sha) + info.size = os.fstat(f.fileno()).st_size + out.addfile(info, f) + sent.add(sha) + meta = json.dumps(files, sort_keys=True).encode() + info = tarfile.TarInfo("manifest.json") + info.size = len(meta) + out.addfile(info, io.BytesIO(meta)) +sys.stdout.buffer.flush() +print(hashlib.sha256(meta).hexdigest(), file=sys.stderr) +""" + + +def native_command(root: str, python: str) -> str: + """Byte-identical to the command NetworkRuntime.rescue builds (orchestrate.py:2546-2550).""" + return ( + f"cd {shlex.quote(root)} && env PYTHONPATH=src {shlex.quote(python)} " + "experiments/gpu_network_v2/orchestrate.py export out - " + "experiments/gpu_network_v2/profile.json" + ) + + +def command_mode(command: str, root: str, python: str) -> tuple[str, list[str]] | None: + """(mode, known) of a native export command: ("full", []), ("hashes", []) or + ("dedup", [KNOWN_SHA256 ...]) as NetworkRuntime.rescue appends them; None otherwise.""" + base = native_command(root, python) + if command == base: + return "full", [] + words = command.removeprefix(base + " ").split(" ") if command.startswith(base + " ") else [] + if words == ["hashes"]: + return "hashes", [] + if words[:1] == ["dedup"] and all(HEX.fullmatch(k) for k in words[1:]): + if " ".join([base, *words]) == command: + return "dedup", words[1:] + return None + + +def checked_manifest(raw: bytes, reject: Callable[[str], Exception]) -> dict: + files = json.loads(raw) + if not isinstance(files, dict) or not 0 < len(files) <= MAX_FILES: + raise reject("network_rescue_dedup_manifest_shape") + for name, want in files.items(): + path = PurePosixPath(name) + if ( + not isinstance(name, str) + or not name + or "\\" in name + or path.is_absolute() + or str(path) != name + or any(part in ("", ".", "..") for part in path.parts) + or not isinstance(want, dict) + or set(want) != {"bytes", "sha256"} + or type(want["bytes"]) is not int + or want["bytes"] < 0 + or not isinstance(want["sha256"], str) + or not HEX.fullmatch(want["sha256"]) + ): + raise reject("network_rescue_dedup_manifest_entry") + return files + + +def make_stream(orch, root: str, python: str, retained: int): + """Drop-in for orchestrate.stream_bounded with the same arguments and failure contract.""" + reject = orch.Reject + native_stream = orch.stream_bounded + + def stream(argv: list[str], archive: Path, log: Path, bound: int, timeout: float) -> None: + mode = command_mode(argv[-1], root, python) + if mode is None: + raise reject("network_rescue_dedup_unexpected_command") + if mode[0] != "full": + # hashes / dedup exports already carry each needed content once: native stream. + return native_stream(argv, archive, log, bound, timeout) + end = time.monotonic() + timeout + blobs = archive.parent / (archive.name + ".blobs") + blobs.mkdir(mode=0o700) + try: + with log.open("wb") as err: + proc = subprocess.Popen( # noqa: S603 (pinned argv, only the command differs) + [*argv[:-1], f"cd {shlex.quote(root)} && {shlex.quote(python)} -"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=err, + start_new_session=True, + ) + + def kill() -> None: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + timer = threading.Timer(timeout, kill) + timer.start() + meta, sizes, received, ok = None, {}, 0, False + try: + assert proc.stdin is not None and proc.stdout is not None + proc.stdin.write(PRODUCER.encode()) + proc.stdin.close() + with tarfile.open(fileobj=proc.stdout, mode="r|") as source: + for member in source: + if meta is not None or not member.isreg(): + raise reject("network_rescue_dedup_member") + received += member.size + if received > bound: + raise reject("network_rescue_archive_budget_exceeded") + data = source.extractfile(member) + assert data is not None + if member.name == "manifest.json": + if member.size > MANIFEST_MAX: + raise reject("rescue_manifest_missing_or_oversized") + meta = data.read(MANIFEST_MAX + 1) + continue + sha = member.name.removeprefix("blobs/") + if member.name != "blobs/" + sha or not HEX.fullmatch(sha): + raise reject("network_rescue_dedup_member") + if sha in sizes or len(sizes) >= MAX_FILES: + raise reject("network_rescue_dedup_member") + digest, size = hashlib.sha256(), 0 + with (blobs / sha).open("xb") as sink: + while block := data.read(1 << 22): + if time.monotonic() > end: + raise reject("network_rescue_deadline_exceeded") + digest.update(block) + sink.write(block) + size += len(block) + if digest.hexdigest() != sha or size != member.size: + raise reject("rescue_content_hash_mismatch") + sizes[sha] = size + ok = True + except (tarfile.ReadError, EOFError) as exc: # stream cut (kill/ssh drop) + late = time.monotonic() > end + raise reject( + "network_rescue_deadline_exceeded" + if late + else "network_rescue_stream_truncated" + ) from exc + finally: + if not ok: + kill() + proc.stdout.close() if proc.stdout else None + rc = proc.wait() + timer.cancel() + if rc: + raise reject("network_export_failed") + if meta is None or len(meta) > MANIFEST_MAX: + raise reject("rescue_manifest_missing_or_oversized") + files = checked_manifest(meta, reject) + if sum(w["bytes"] for w in files.values()) > retained: + raise reject("actual_artifact_budget_exceeded") + if {w["sha256"]: w["bytes"] for w in files.values()} != sizes: + raise reject("network_rescue_dedup_blob_set") + with archive.open("xb") as sink, tarfile.open(fileobj=sink, mode="w") as out: + for name in sorted(files): + if time.monotonic() > end: + raise reject("network_rescue_deadline_exceeded") + info = tarfile.TarInfo("out/" + name) + info.size, info.mode = files[name]["bytes"], 0o600 + with (blobs / files[name]["sha256"]).open("rb") as blob: + out.addfile(info, blob) + info = tarfile.TarInfo("manifest.json") + info.size, info.mode = len(meta), 0o600 + out.addfile(info, __import__("io").BytesIO(meta)) + out.close() + sink.flush() + os.fsync(sink.fileno()) + finally: + shutil.rmtree(blobs, ignore_errors=True) + + return stream + + +def load_orchestrate(tree: Path): + """Same module file the native cleanup loads (launcher.py:720-730).""" + spec = importlib.util.spec_from_file_location( + "network_cleanup_runtime", tree / "experiments/gpu_network_v2/orchestrate.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--role", choices=("h0", "h1"), required=True) + tree = os.environ.get("HT_TREE") or sys.exit("HT_TREE must name the controller tree") + parser.add_argument("--tree", type=Path, default=Path(tree)) + args = parser.parse_args(argv) + from hypertrain.gpu_ops.journal import Journal + from hypertrain.gpu_ops.launcher import Orchestrator + from hypertrain.gpu_ops.provider import Provider, read_api_key + + cfg = json.loads((args.run_dir / "config.json").read_text()) + journal = Journal(args.run_dir) + opened = journal.last("transaction_open") + if cfg.get("cleanup_contract") != "network-v2" or opened is None: + raise SystemExit("not a network-v2 transaction") + provider = Provider( + str(cfg["base_url"]), + read_api_key(str(cfg["key_file"])), + args.run_dir, + journal, + bool(opened["live"]), + ) + lifecycle = Orchestrator(dict(cfg), journal, provider, args.run_dir, actor="rescue-dedup") + if lifecycle.cleanup_started() or lifecycle.receipt(args.role) is None: + raise SystemExit("cleanup started or role not owned") + orch = load_orchestrate(args.tree) + profile = json.loads(Path(cfg["network_profile_file"]).read_bytes()) + runtime = orch.NetworkRuntime(lifecycle, profile) + python = lifecycle.cfg.get("remote_python", profile["runtime"]["remote_python"]) + retained = orch.artifact_budget(profile)["retained_bytes_per_host"] + orch.stream_bounded = make_stream(orch, lifecycle.remote_root(args.role), python, retained) + row = runtime.rescue(args.role, fresh=True) + print( + json.dumps( + { + "role": args.role, + "generation": row["generation"], + "tar": row["tar"], + "tar_sha256": row["tar_sha256"], + "files": len(row["files"]), + "elapsed_seconds": round(row["elapsed_seconds"], 1), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ops/rescue_parallel.py b/scripts/ops/rescue_parallel.py new file mode 100644 index 0000000..922ca56 --- /dev/null +++ b/scripts/ops/rescue_parallel.py @@ -0,0 +1,473 @@ +"""Native network-v2 rescue over N parallel channels (run-m4 Modal recovery adapter). + +Same contract as rescue_dedup.py: runs the unchanged NetworkRuntime.rescue(role, fresh=True) +(same generation, pinned SSH, bound, deadline, verify_export, remote manifest digest check and +native network_rescued row). Only stream_bounded is replaced in this process: + 1. pinned SSH: host hashes out/ and returns the manifest; its digest goes to the native log. + 2. distinct blobs split over N workers; each worker streams a tar of blobs either over its + own pinned SSH channel (--via ssh) or as a direct TCP connection from the host to this + controller (--via tcp, launched over pinned SSH, authenticated by a one-time token). + 3. every blob is sha256-checked against the SSH-delivered manifest, then the exact export + tar is rebuilt locally. Integrity is anchored on the pinned SSH channel, never on TCP. +""" + +# ruff: noqa: E501, S104 (verbatim controller tool: rescue listeners bind all interfaces) +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +import secrets +import shlex +import signal +import socket +import subprocess +import sys +import tarfile +import threading +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from rescue_dedup import ( # noqa: E402 + HEX, + MANIFEST_MAX, + checked_manifest, + command_mode, + load_orchestrate, +) + +MANIFEST = r""" +import hashlib, json, os, sys +files = {} +for dirpath, dirs, names in os.walk("out"): + dirs.sort() + for d in dirs: + if os.path.islink(os.path.join(dirpath, d)): + sys.exit("export_symlink_rejected " + os.path.join(dirpath, d)) + for name in sorted(names): + path = os.path.join(dirpath, name) + if os.path.islink(path) or not os.path.isfile(path): + sys.exit("export_symlink_rejected " + path) + digest = hashlib.sha256() + with open(path, "rb") as f: + while block := f.read(1 << 22): + digest.update(block) + size = f.tell() + files[os.path.relpath(path, "out")] = {"bytes": size, "sha256": digest.hexdigest()} +meta = json.dumps(files, sort_keys=True).encode() +sys.stdout.buffer.write(meta) +sys.stdout.buffer.flush() +print(hashlib.sha256(meta).hexdigest(), file=sys.stderr) +""" + +RANGES = r""" +import json, os, socket +spec = json.loads(SPEC) +s = socket.socket() +s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1 << 25) +s.settimeout(30) +s.connect((spec["host"], spec["port"])) +s.settimeout(None) +s.sendall(spec["token"].encode() + b"\n") +for path, offset, length in spec["chunks"]: + with open(os.path.join("out", path), "rb") as f: + sent = 0 + while sent < length: + n = s.sendfile(f, offset + sent, length - sent) + if not n: + raise SystemExit("short_read " + path) + sent += n +s.shutdown(socket.SHUT_WR) +s.close() +""" + +CHUNK = 64 << 20 + +SENDER = r""" +import json, os, socket, sys, tarfile +spec = json.loads(SPEC) +if spec["via"] == "tcp": + s = socket.create_connection((spec["host"], spec["port"]), timeout=30) + s.settimeout(None) + s.sendall(spec["token"].encode() + b"\n") + sink = s.makefile("wb", buffering=1 << 22) +else: + sink = sys.stdout.buffer +with tarfile.open(fileobj=sink, mode="w|", format=tarfile.PAX_FORMAT) as out: + for sha, path in spec["items"]: + with open(os.path.join("out", path), "rb") as f: + info = tarfile.TarInfo("blobs/" + sha) + info.size = os.fstat(f.fileno()).st_size + out.addfile(info, f) +sink.flush() +if spec["via"] == "tcp": + s.shutdown(socket.SHUT_WR) + s.close() +""" + + +def make_stream( + orch, + root: str, + python: str, + retained: int, + via: str, + workers: int, + public_ip: str, + base_port: int, +): + reject = orch.Reject + + native_stream = orch.stream_bounded + + def stream(argv: list[str], archive: Path, log: Path, bound: int, timeout: float) -> None: + # Hash-only (" hashes") and dedup (" dedup [sha ...]") exports: the native single stream + # already carries each needed content once with its exact tar shape; keep it untouched. + mode = command_mode(argv[-1], root, python) + if mode is None: + raise reject("network_rescue_dedup_unexpected_command") + if mode[0] != "full": + return native_stream(argv, archive, log, bound, timeout) + end = time.monotonic() + timeout + prefix = argv[:-1] + remote = f"cd {shlex.quote(root)} && {shlex.quote(python)} -" + procs: list[subprocess.Popen] = [] + lock = threading.Lock() + + def spawn(script: str, out=subprocess.PIPE, err=subprocess.PIPE) -> subprocess.Popen: + p = subprocess.Popen( # noqa: S603 (pinned argv, only the command differs) + [*prefix, remote], + stdin=subprocess.PIPE, + stdout=out, + stderr=err, + start_new_session=True, + ) + with lock: + procs.append(p) + assert p.stdin is not None + p.stdin.write(script.encode()) + p.stdin.close() + return p + + def kill_all() -> None: + with lock: + for p in procs: + try: + import os + + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + + timer = threading.Timer(timeout, kill_all) + timer.start() + blobs = archive.parent / (archive.name + ".blobs") + blobs.mkdir(mode=0o700) + try: + p = spawn(MANIFEST) + assert p.stdout is not None and p.stderr is not None + meta = p.stdout.read(MANIFEST_MAX + 1) + err = p.stderr.read(65536) + p.wait() + log.write_bytes(err) + if p.returncode: + raise reject("network_export_failed") + if len(meta) > MANIFEST_MAX: + raise reject("rescue_manifest_missing_or_oversized") + files = checked_manifest(meta, reject) + if sum(w["bytes"] for w in files.values()) > retained: + raise reject("actual_artifact_budget_exceeded") + distinct: dict[str, tuple[str, int]] = {} + for name in sorted(files): + distinct.setdefault(files[name]["sha256"], (name, files[name]["bytes"])) + if sum(n for _, n in distinct.values()) > bound: + raise reject("network_rescue_archive_budget_exceeded") + errors: list[BaseException] = [] + if via == "range": + import os + + chunks = [] + for sha, (name, size) in sorted(distinct.items()): + with (blobs / sha).open("xb") as f: + f.truncate(size) + chunks += [(sha, name, o, min(CHUNK, size - o)) for o in range(0, size, CHUNK)] + parts: list[list[tuple[str, str, int, int]]] = [[] for _ in range(workers)] + for k, c in enumerate(chunks): + parts[k % workers].append(c) + parts = [p for p in parts if p] + + def ranged(i: int, part: list[tuple[str, str, int, int]]) -> None: + try: + token = secrets.token_hex(32) + srv = socket.create_server(("0.0.0.0", base_port + i)) + srv.settimeout(max(1.0, min(120.0, end - time.monotonic()))) + try: + spec = { + "host": public_ip, + "port": base_port + i, + "token": token, + "chunks": [(n, o, ln) for _, n, o, ln in part], + } + p = spawn( + "SPEC = " + repr(json.dumps(spec)) + "\n" + RANGES, + out=subprocess.DEVNULL, + ) + for _ in range(8): + conn, _addr = srv.accept() + conn.settimeout(max(1.0, end - time.monotonic())) + f = conn.makefile("rb", buffering=1 << 22) + if f.readline(80).strip() == token.encode(): + break + f.close() + conn.close() + else: + raise reject("network_rescue_stream_unauthenticated") + try: + for sha, _n, off, ln in part: + fd = os.open(blobs / sha, os.O_WRONLY) + try: + got = 0 + while got < ln: + if time.monotonic() > end: + raise reject("network_rescue_deadline_exceeded") + block = f.read(min(1 << 22, ln - got)) + if not block: + raise reject("network_rescue_stream_truncated") + os.pwrite(fd, block, off + got) + got += len(block) + finally: + os.close(fd) + if f.read(1): + raise reject("network_rescue_dedup_member") + finally: + f.close() + conn.close() + if p.wait(timeout=max(1.0, end - time.monotonic())) != 0: + raise reject("network_export_failed") + finally: + srv.close() + except OSError as exc: + late = time.monotonic() > end + errors.append( + reject( + "network_rescue_deadline_exceeded" + if late + else "network_rescue_stream_truncated" + ) + ) + errors[-1].__cause__ = exc + kill_all() + except BaseException as exc: # noqa: BLE001 + errors.append(exc) + kill_all() + + threads = [ + threading.Thread(target=ranged, args=(i, p)) for i, p in enumerate(parts) + ] + for t in threads: + t.start() + for t in threads: + t.join() + if errors: + raise errors[0] + for sha, (_name, size) in distinct.items(): + if time.monotonic() > end: + raise reject("network_rescue_deadline_exceeded") + with (blobs / sha).open("rb") as f: + if hashlib.file_digest(f, "sha256").hexdigest() != sha or f.tell() != size: + raise reject("rescue_content_hash_mismatch") + groups: list[list[tuple[str, str]]] = [[] for _ in range(workers)] + loads = [0] * workers + for sha, (name, size) in sorted(distinct.items(), key=lambda kv: -kv[1][1]): + i = loads.index(min(loads)) + groups[i].append((sha, name)) + loads[i] += size + groups = [g for g in groups if g] if via != "range" else [] + + def receive(source, want: set[str]) -> None: + got = set() + with tarfile.open(fileobj=source, mode="r|") as tar: + for member in tar: + sha = member.name.removeprefix("blobs/") + if ( + not member.isreg() + or member.name != "blobs/" + sha + or not HEX.fullmatch(sha) + or sha not in want + or sha in got + ): + raise reject("network_rescue_dedup_member") + data = tar.extractfile(member) + assert data is not None + digest, size = hashlib.sha256(), 0 + with (blobs / sha).open("xb") as sink: + while block := data.read(1 << 22): + if time.monotonic() > end: + raise reject("network_rescue_deadline_exceeded") + digest.update(block) + sink.write(block) + size += len(block) + if digest.hexdigest() != sha or size != distinct[sha][1]: + raise reject("rescue_content_hash_mismatch") + got.add(sha) + if got != want: + raise reject("network_rescue_dedup_blob_set") + + def worker(i: int, items: list[tuple[str, str]]) -> None: + try: + spec = {"via": via, "items": items} + want = {sha for sha, _ in items} + if via == "ssh": + p = spawn("SPEC = " + repr(json.dumps(spec)) + "\n" + SENDER) + assert p.stdout is not None + try: + receive(p.stdout, want) + finally: + if p.wait() != 0: + raise reject("network_export_failed") + return + token = secrets.token_hex(32) + srv = socket.create_server(("0.0.0.0", base_port + i), reuse_port=False) + srv.settimeout(max(1.0, min(120.0, end - time.monotonic()))) + try: + spec.update(host=public_ip, port=base_port + i, token=token) + p = spawn( + "SPEC = " + repr(json.dumps(spec)) + "\n" + SENDER, + out=subprocess.DEVNULL, + ) + for _ in range(8): # ignore stray connections without the token + conn, _addr = srv.accept() + conn.settimeout(max(1.0, end - time.monotonic())) + f = conn.makefile("rb", buffering=1 << 22) + if f.readline(80).strip() == token.encode(): + break + f.close() + conn.close() + else: + raise reject("network_rescue_stream_unauthenticated") + try: + receive(f, want) + finally: + f.close() + conn.close() + if p.wait(timeout=max(1.0, end - time.monotonic())) != 0: + raise reject("network_export_failed") + finally: + srv.close() + except (tarfile.ReadError, EOFError, OSError) as exc: + late = time.monotonic() > end + errors.append( + reject( + "network_rescue_deadline_exceeded" + if late + else "network_rescue_stream_truncated" + ) + ) + errors[-1].__cause__ = exc + kill_all() + except BaseException as exc: # noqa: BLE001 (first error re-raised below) + errors.append(exc) + kill_all() + + threads = [threading.Thread(target=worker, args=(i, g)) for i, g in enumerate(groups)] + for t in threads: + t.start() + for t in threads: + t.join() + if errors: + raise errors[0] + with archive.open("xb") as sink, tarfile.open(fileobj=sink, mode="w") as out: + for name in sorted(files): + if time.monotonic() > end: + raise reject("network_rescue_deadline_exceeded") + info = tarfile.TarInfo("out/" + name) + info.size, info.mode = files[name]["bytes"], 0o600 + with (blobs / files[name]["sha256"]).open("rb") as blob: + out.addfile(info, blob) + info = tarfile.TarInfo("manifest.json") + info.size, info.mode = len(meta), 0o600 + out.addfile(info, io.BytesIO(meta)) + out.close() + sink.flush() + import os + + os.fsync(sink.fileno()) + finally: + timer.cancel() + kill_all() + import shutil + + shutil.rmtree(blobs, ignore_errors=True) + + return stream + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-dir", type=Path, required=True) + parser.add_argument("--role", choices=("h0", "h1"), required=True) + tree = os.environ.get("HT_TREE") or sys.exit("HT_TREE must name the controller tree") + parser.add_argument("--tree", type=Path, default=Path(tree)) + parser.add_argument("--via", choices=("ssh", "tcp", "range"), required=True) + parser.add_argument("--workers", type=int, default=4) + parser.add_argument("--public-ip", default="") + parser.add_argument("--base-port", type=int, default=18560) + args = parser.parse_args(argv) + if args.via in ("tcp", "range") and not args.public_ip: + raise SystemExit("--public-ip required for --via tcp") + from hypertrain.gpu_ops.journal import Journal + from hypertrain.gpu_ops.launcher import Orchestrator + from hypertrain.gpu_ops.provider import Provider, read_api_key + + cfg = json.loads((args.run_dir / "config.json").read_text()) + journal = Journal(args.run_dir) + opened = journal.last("transaction_open") + if cfg.get("cleanup_contract") != "network-v2" or opened is None: + raise SystemExit("not a network-v2 transaction") + provider = Provider( + str(cfg["base_url"]), + read_api_key(str(cfg["key_file"])), + args.run_dir, + journal, + bool(opened["live"]), + ) + lifecycle = Orchestrator(dict(cfg), journal, provider, args.run_dir, actor="rescue-parallel") + if lifecycle.cleanup_started() or lifecycle.receipt(args.role) is None: + raise SystemExit("cleanup started or role not owned") + orch = load_orchestrate(args.tree) + profile = json.loads(Path(cfg["network_profile_file"]).read_bytes()) + runtime = orch.NetworkRuntime(lifecycle, profile) + python = lifecycle.cfg.get("remote_python", profile["runtime"]["remote_python"]) + retained = orch.artifact_budget(profile)["retained_bytes_per_host"] + orch.stream_bounded = make_stream( + orch, + lifecycle.remote_root(args.role), + python, + retained, + args.via, + args.workers, + args.public_ip, + args.base_port, + ) + row = runtime.rescue(args.role, fresh=True) + print( + json.dumps( + { + "role": args.role, + "generation": row["generation"], + "tar": row["tar"], + "tar_sha256": row["tar_sha256"], + "files": len(row["files"]), + "elapsed_seconds": round(row["elapsed_seconds"], 1), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ops/task8-deps.py b/scripts/ops/task8-deps.py new file mode 100644 index 0000000..f01958d --- /dev/null +++ b/scripts/ops/task8-deps.py @@ -0,0 +1,270 @@ +"""Fresh task-8 hosts: controller boot/attach/trust, one pinned dependency batch, seed pre-stage. + +Runs between admit and qualification on the admitted run dir. Every SSH call is the controller's +DeadlineSsh over its pinned known_hosts, capped at the signed cutoff minus the full qualification +window (Q + hard margin); a failure on one host cancels the other at its next SSH boundary. +""" + +import importlib.util +import json +import os +import re +import shlex +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import hf_seed # noqa: E402 + +from hypertrain.gpu_ops.remote import RemoteError, TrustError, ensure_identity # noqa: E402 + +TREE = Path(os.environ.get("HT_TREE") or sys.exit("HT_TREE must name the controller tree")) +spec = importlib.util.spec_from_file_location( + "orchestrate", TREE / "experiments/gpu_network_v2/orchestrate.py" +) +assert spec is not None and spec.loader is not None +cli = importlib.util.module_from_spec(spec) +sys.modules["orchestrate"] = cli +spec.loader.exec_module(cli) + +RUN = Path(sys.argv[1]) +BASE = json.loads(Path(sys.argv[2]).read_bytes())["record"] +FILES = {k: Path(v["path"]) for k, v in BASE["files"].items()} +PROFILE = json.loads(FILES["profile"].read_bytes()) +JOB = json.loads(FILES[BASE["role_inputs"]["job"]].read_bytes()) +UPLOADS = {"job.json": FILES[BASE["role_inputs"]["job"]]} | { + JOB["object_paths"][k]: FILES[v] for k, v in BASE["role_inputs"]["objects"].items() +} +if any(not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", rel) for rel in UPLOADS): + raise SystemExit("seed object paths must be plain file names") +GPUS = cli.admission_limits(PROFILE)["gpus"] +PINS = ( + "fastapi==0.142.2 uvicorn==0.54.0 httpx==0.28.1 pydantic==2.13.5 py-sr25519-bindings==0.2.4 " + "numpy==2.5.3 safetensors==0.8.0 py-ecc==8.0.0 pyarrow==25.0.1 tokenizers==0.23.2" +) +PROBE = ( + "import pathlib,torch,fastapi,uvicorn,httpx,pydantic,sr25519,numpy,safetensors,py_ecc," + "pyarrow,tokenizers; " + "assert pathlib.Path(torch.__file__).resolve().is_relative_to('/venv/main'); " + "assert torch.__version__=='2.14.0+cu130' and torch.version.cuda=='13.0'; " + f"n=torch.cuda.device_count(); assert torch.cuda.is_available() and n=={GPUS}; " + f"assert all(torch.cuda.get_device_properties(i).multi_processor_count==" + f"{PROFILE['admission']['sm_count']} for i in range(n)); " + "print('DEPS_OK',n,torch.__version__)" +) +VENV = "/opt/hypertrain/venv/bin/python" +# hypertrain.models.opendecision imports the external opendecision-arch-20261008 package; the +# remote venv gets the exact trusted copy (/data/pydeps == /root/distributed-decision-training/ +# opendecision-arch-20261008/src/opendecision, same 15 file hashes) on a .pth, never pip/index. +ARCH = Path("/data/pydeps/opendecision") +ARCH_REMOTE = "/opt/hypertrain/opendecision-arch-20261008/src" +ARCH_SHA256 = { + "__init__.py": "269301f53d89a0f09b3d58a2a321544ccdf88ecef1e1a8b9bb5ed883b3aea371", + "calibrate.py": "d0ae39c24c13469030f06b33a2701dce063b6be53588e48c41fc2009636ada91", + "cascade.py": "c3d747ec1c27aa5bffe32f6bc813ee7c76d7ce428b3224afe853925f0e9e540e", + "coherence.py": "f610e7e5ad9c47a75e6349651157a6f0e40fd773b5fca8d9c1539127f836998a", + "decider.py": "9a85968f3a736f2a273c73768e0772aeef145afa69399821487cb05ba62b892a", + "losses.py": "292a7b7e0f1466c1ddbd8153154f215da5fbb797bffffd9c976c09f4e9f35f02", + "masking.py": "a785e1589cef654a898417c73928181eeb794fb5e928db2f38eb66eca7bd4e79", + "model.py": "4821c2e22c19a033ee8ae4932d96fcde88c9ac2601efe42d9380968a5a8b9a8e", + "presets.py": "5d0e1dcae90b5f3c0808d08c4ae4c34582ca7dfb4397661d907c7e8af90af1cd", + "records.py": "2c11f4c91c99ebc7db65098ca6c8632df7e6883397a96a5df013a4aff37b3281", + "schema.py": "4ae15a66b4228a6691a5b2854535aa80251c9a1ad74b98275baca454899f5787", + "serve.py": "c81cb0bdc6e6cf4665b0034f61705757885a70af45baea2b1a3d5317ad27d9e7", + "teacher.py": "57ac31ffd014c95eca1dfa071bec0952e7776ac1f9a205064e6c8f9a167acf26", + "tokenizer.py": "5422c7bfd66cf13c794473ac73619c8513d126cfc35556af09786d8321efe3bc", + "train.py": "e92cc2708d6bc7ebd4600a8bd588071e50090ff632d1e2623cfe656fd8065b7a", +} +if {p.name for p in ARCH.glob("*.py")} != set(ARCH_SHA256) or any( + cli.fsha(ARCH / name) != digest for name, digest in ARCH_SHA256.items() +): + raise SystemExit("opendecision arch source does not match the pinned hashes") +ARCH_LINK = ( + f"set -e; d=$({VENV} -c 'import sysconfig; print(sysconfig.get_paths()[\"purelib\"])'); " + f'echo {ARCH_REMOTE} > "$d/zz-opendecision-arch.pth"; ' + f"{VENV} -c 'import opendecision.model, opendecision.masking; print(opendecision.__file__)'" +) +INSTALL = ( + f"set -e; [ -x {VENV} ] || " + "/venv/main/bin/python -m venv --system-site-packages /opt/hypertrain/venv; " + f"{VENV} -m pip install --no-input --disable-pip-version-check -q {PINS}; " + f"{VENV} -c {shlex.quote(PROBE)}" +) + +cfg = json.loads((RUN / "config.json").read_bytes()) +j = cli.Journal(RUN) +window = j.last("fresh_window") +if window is None or {r["role"] for r in j.all("receipt")} != {"h0", "h1"}: + raise SystemExit("fresh admitted run with h0/h1 receipts required") +provider = cli.Provider(cfg["base_url"], cli.read_api_key(cfg["key_file"]), RUN, j, True) +orch = cli.Orchestrator(cfg, j, provider, RUN) +reserve = ( + PROFILE["microqualification"]["qualification_deadline_seconds"] + + PROFILE["schedule"]["hard_margin_seconds"] +) +CUTOFF = min(orch.deadline(), window["cutoff_unix"]) - reserve +LOGS = RUN / "logs" +CANCEL = threading.Event() + + +PUT_STREAMS, PUT_SPLIT = 16, 256 << 20 + + +def put(ssh, local: Path, remote: str, attempts: int = 5) -> None: + # One direct TCP stream measured 21 Mbit/s on some Vast hosts (v3c h1): large files go as + # PUT_STREAMS parallel byte ranges, joined remotely; staged_match then proves size+sha256. + size = local.stat().st_size + if size > PUT_SPLIT: + return put_ranges(ssh, local, remote, size, attempts) + return put_one(ssh, local, remote, attempts) + + +def put_ranges(ssh, local: Path, remote: str, size: int, attempts: int) -> None: + step = -(-size // PUT_STREAMS) + q = shlex.quote(remote) + + def part(i: int) -> None: + with local.open("rb") as f: + f.seek(i * step) + data = f.read(step) + for attempt in range(attempts): + try: + ssh._stream(f"cat > {q}.r{i:02d}", "put-range", data) + return + except RemoteError as e: + if "rc=255" not in str(e) or attempt == attempts - 1 or CANCEL.is_set(): + raise + CANCEL.wait(3) + + n = -(-size // step) + with ThreadPoolExecutor(n) as pool: + list(pool.map(part, range(n))) + joined = " ".join(f"{q}.r{i:02d}" for i in range(n)) + ssh.timeout = 600 + if ssh.run( + f"cat {joined} > {q}.part && rm -f {joined} && mv {q}.part {q}", "put-join", LOGS + ).returncode: + raise RuntimeError(f"put_join_failed {remote}") + + +def put_one(ssh, local: Path, remote: str, attempts: int) -> None: + # Modal TCP tunnel resets some fresh SSH handshakes (rc 255, kex reset); retry transport only. + for attempt in range(attempts): + try: + return ssh.put(local, remote) + except RemoteError as e: + if "rc=255" not in str(e) or attempt == attempts - 1: + raise + CANCEL.wait(3) + + +# Seed objects that are header + zeros + genesis theta are rebuilt on the host from the pinned HF +# checkpoint (controller ingress is not the bottleneck there); only the header crosses the link. +HF_CKPT = "/root/hf-genesis.safetensors" # outside the run root rescue/audit read +HF_PLANS = { + rel: hf_seed.plan(path) for rel, path in UPLOADS.items() if path.stat().st_size > PUT_SPLIT +} + + +def hf_build(ssh, role: str, local: Path, remote: str, plan, ckpt: str) -> bool: + head, copy = plan + try: + ssh.timeout = 1800 + if copy and ssh.run(hf_seed.download_cmd(ckpt), "hf-download", LOGS).returncode: + print(role, "HF_DOWNLOAD_FAILED", flush=True) + return False + ssh.timeout = 600 + ssh._stream( + f"{VENV} -c " + shlex.quote(hf_seed.BUILD), + "hf-build", + hf_seed.payload(head, copy, remote, ckpt, local.stat().st_size), + ) + return cli.staged_match(ssh, remote, local, LOGS) + except (RemoteError, OSError) as e: + print(role, "HF_BUILD_FAILED", remote, e, flush=True) + return False + + +def prepare(role: str) -> str: + try: + orch.boot(role) + orch.attach(role) + # Signed config pins keyscan_attempts=1 and sshd starts after "running": retry the whole + # controller trust (0 keys writes no known_hosts) until sshd answers or CUTOFF. + while True: + try: + orch.trust(role) + break + except TrustError as e: + if "returned 0 distinct" not in str(e) or time.time() + 15 > CUTOFF: + raise + CANCEL.wait(15) + if CANCEL.is_set(): + raise + ssh = cli.deadline_ssh(orch, role, CUTOFF) + ssh.cancel = CANCEL + ssh.timeout = 1200 + installed = ssh.run(INSTALL, "deps-install", LOGS) + # The provider proxy may not yet recognize the just-attached key. + while ( + installed.returncode == 255 + and "Permission denied (publickey)" in installed.stderr + and time.time() + 15 < CUTOFF + ): + CANCEL.wait(15) + if CANCEL.is_set(): + break + installed = ssh.run(INSTALL, "deps-install", LOGS) + if installed.returncode or not installed.stdout.strip().startswith(f"DEPS_OK {GPUS} "): + raise RuntimeError(f"{role} deps_install_failed rc={installed.returncode}") + print(role, installed.stdout.strip(), flush=True) + ssh.timeout = 60 + if ssh.run(f"mkdir -p {ARCH_REMOTE}/opendecision", "deps-arch-mkdir", LOGS).returncode: + raise RuntimeError(f"{role} arch_mkdir_failed") + for name in sorted(ARCH_SHA256): + remote = f"{ARCH_REMOTE}/opendecision/{name}" + if not cli.staged_match(ssh, remote, ARCH / name, LOGS): + put(ssh, ARCH / name, remote) + if not cli.staged_match(ssh, remote, ARCH / name, LOGS): + raise RuntimeError(f"{role} arch_transport_size_or_hash {name}") + linked = ssh.run(ARCH_LINK, "deps-arch-link", LOGS) + if linked.returncode or linked.stdout.strip() != f"{ARCH_REMOTE}/opendecision/__init__.py": + raise RuntimeError(f"{role} arch_import_failed rc={linked.returncode}") + print(role, "ARCH_OK", len(ARCH_SHA256), flush=True) + seed = orch.remote_root(role) + "/out/seed" + ssh.timeout = 60 + if ssh.run(f"mkdir -p {shlex.quote(seed)}", "deps-mkdir", LOGS).returncode: + raise RuntimeError(f"{role} seed_mkdir_failed") + for rel, path in sorted(UPLOADS.items(), key=lambda kv: -kv[1].stat().st_size): + ssh.timeout = 600 + if cli.staged_match(ssh, seed + "/" + rel, path, LOGS): + print(role, "REUSED", rel, flush=True) + continue + plan = HF_PLANS.get(rel) + if plan and hf_build(ssh, role, path, seed + "/" + rel, plan, HF_CKPT): + print(role, "HF_BUILT", rel, hf_seed.REVISION, flush=True) + continue + ssh.timeout = 7200 + put(ssh, path, seed + "/" + rel) + ssh.timeout = 600 + if not cli.staged_match(ssh, seed + "/" + rel, path, LOGS): + raise RuntimeError(f"{role} seed_transport_size_or_hash {rel}") + print(role, "STAGED", rel, path.stat().st_size, flush=True) + ssh.timeout = 60 + ssh.run(f"rm -f {HF_CKPT}* {shlex.quote(seed)}/*.hf-part", "hf-cleanup", LOGS) + return f"{role} DEPS_AND_SEED_READY" + except BaseException: + CANCEL.set() + raise + + +# Direct ssh makes h0/h1 ready in the same second: both attach() threads would race the +# unlocked ensure_identity() ssh-keygen. Create the run key once, before the pool. +ensure_identity(RUN) + +with ThreadPoolExecutor(2) as pool: + for line in pool.map(prepare, ("h0", "h1")): + print(line, flush=True) diff --git a/scripts/ops/task8-materialize-roots.py b/scripts/ops/task8-materialize-roots.py new file mode 100644 index 0000000..dcc5649 --- /dev/null +++ b/scripts/ops/task8-materialize-roots.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +"""Materialize byte-exact /data/hypertrain-{role} roots from verified local custody. + +Offline, read-only on custody. Native order (orchestrate continue): rescue -> artifact +manifest check -> record_qualification, so the accepted network_qualification_done must +bind (artifact_manifest_sha256) to the LAST network_rescued that precedes it. Sources: that +rescue tar (sha + verify_export), base review record (bundle/profile/registry/sources sha), +run-dir cli-qualification-{role}.json (copied byte-identical, never rewritten) and +cli-receipt-{role}.json. Every byte is hashed while streaming; regular files only; no +absolute, '..', duplicate or link members. Builds .partial-, renames only after +all checks; if native check_inputs then rejects, removes only the root this call published +and writes materialize-failed-{role}.json, so a retry starts clean. +usage: task8-materialize-roots.py RUN_DIR REVIEW_BASE ROLE [ROLE...]""" + +import hashlib +import json +import os +import shutil +import sys +import tarfile +from pathlib import Path + +_TREE = os.environ.get("HT_TREE") or sys.exit("HT_TREE must name the controller tree") +sys.path[:0] = [f"{_TREE}/src", f"{_TREE}/experiments/gpu_network_v2"] +import orchestrate as orch # noqa: E402 + +ROOT_PREFIX = "/data/hypertrain-" +from hypertrain.gpu_ops import network_qualification as nq # noqa: E402 +from hypertrain.gpu_ops.journal import durable_write, fsha # noqa: E402 + +CHUNK = 1 << 20 + + +class Refused(Exception): + pass + + +def safe(rel: str) -> Path: + path = Path(rel) + if not rel or path.is_absolute() or ".." in path.parts or "\\" in rel: + raise Refused("unsafe_path " + rel) + return path + + +def put(source, member, target: Path, sha: str, size: int) -> None: + if not member.isfile() or member.size != size: + raise Refused("member_not_regular_or_size " + member.name) + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + digest, seen = hashlib.sha256(), 0 + stream = source.extractfile(member) + with target.open("xb") as out: + while block := stream.read(CHUNK): + seen += len(block) + if seen > size: + raise Refused("member_oversize " + member.name) + digest.update(block) + out.write(block) + out.flush() + os.fsync(out.fileno()) + os.chmod(target, 0o600) + if seen != size or digest.hexdigest() != sha: + raise Refused("member_hash " + member.name) + + +def copy(src: Path, target: Path, sha: str) -> None: + data = src.read_bytes() + if hashlib.sha256(data).hexdigest() != sha: + raise Refused("input_hash " + str(src)) + durable_write(target, data) + + +GENERATION_KINDS = ( + "network_execution_intent", + "network_job_started", + "network_job_done", + "network_qualification_captured", + "network_qualification_failed", +) + + +def select(rows: list[dict], role: str) -> tuple[dict | None, dict]: + """Accepted done row when present (native order); otherwise the LAST network_rescued + after this role's network_qualification_captured whose generation equals the native + rescue generation recomputed from the journal. done-absent is reported, never invented.""" + done = [ + i + for i, r in enumerate(rows) + if r["kind"] == "network_qualification_done" and r.get("role") == role + ] + if done: + accepted = rows[done[-1]]["result"] + if accepted.get("passed") is not True or accepted.get("root_verified") is not True: + raise Refused("qualification_not_passed_root_verified") + before = [ + r for r in rows[: done[-1]] if r["kind"] == "network_rescued" and r.get("role") == role + ] + if not before: + raise Refused("no_rescue_before_qualification_done") + rescue = before[-1] + if ( + accepted.get("artifact_manifest_sha256") + != hashlib.sha256(json.dumps(rescue["files"], sort_keys=True).encode()).hexdigest() + ): + raise Refused("accepted_result_not_bound_to_rescue") + return accepted, rescue + captured = [ + i + for i, r in enumerate(rows) + if r["kind"] == "network_qualification_captured" and r.get("role") == role + ] + if not captured: + raise Refused("no_qualification_captured") + terminal = [ + r + for r in rows + if r.get("role") == role + and r["kind"] in ("network_qualification_captured", "network_qualification_failed") + ] + intents = [ + r + for r in rows + if r.get("role") == role + and r["kind"] == "network_execution_intent" + and r.get("phase") == "qualification" + ] + if ( + terminal[-1]["kind"] != "network_qualification_captured" + or len(intents) != 1 + or intents[0].get("executions") != 2 + ): + raise Refused("qualification_capture_or_intent_invalid") + after = [ + r + for r in rows[captured[-1] + 1 :] + if r["kind"] == "network_rescued" and r.get("role") == role + ] + if not after: + raise Refused("no_rescue_after_qualification_captured") + rescue = after[-1] + generation = orch.sha256( + json.dumps( + [r for r in rows if r.get("role") == role and r["kind"] in GENERATION_KINDS], + sort_keys=True, + ).encode() + ) + if rescue.get("generation") != generation: + raise Refused("rescue_generation_differs_from_journal") + return None, rescue + + +def materialize(run: Path, base: dict, role: str) -> dict: + rows = [json.loads(line) for line in (run / "journal.jsonl").read_text().splitlines() if line] + accepted, rescue = select(rows, role) + config_path = run / f"cli-qualification-{role}.json" + config_bytes = config_path.read_bytes() + cfg = json.loads(config_bytes) + root = Path(cfg["tree"]) + if str(root) != ROOT_PREFIX + role or cfg["role"] != role: + raise Refused("config_root_or_role") + expect = { + "profile": root / "profile.json", + "registry_manifest": root / "registry.json", + "lifecycle_receipts": root / "receipts.json", + "seed_job": root / "out/seed/job.json", + } + if any(Path(cfg[k]) != v for k, v in expect.items()): + raise Refused("config_paths_outside_root") + if root.exists() or root.is_symlink(): + raise Refused("destination_exists " + str(root)) + archive = Path(rescue["tar"]) + if fsha(archive) != rescue["tar_sha256"]: + raise Refused("rescue_tar_changed") + profile = json.loads(Path(base["files"]["profile"]["path"]).read_bytes()) + # Re-verify in the row's recorded mode: full rows (no dedup_known) against the full bound. + bound, hashes_only, dedup = orch.rescue_mode(profile, rescue) + files = orch.verify_export(archive, bound, hashes_only, dedup) + carried = set(orch.carried_names(files, hashes_only, dedup)) + if files != rescue["files"]: + raise Refused("rescue_manifest_differs_from_journal") + # Native continue binds accepted result_sha256 to the captured result.json bytes + # (orchestrate 3444); + # here the same bytes must sit in the rescued out/ tree and name this exact config + host. + result_rel = str(Path(cfg["output"]).relative_to(root / "out") / "result.json") + result_sha = files.get(result_rel, {}).get("sha256") + if result_sha is None or accepted is not None and result_sha != accepted.get("result_sha256"): + raise Refused("accepted_result_sha_mismatch") + with tarfile.open(archive) as source: + captured = json.loads(source.extractfile("out/" + result_rel).read()) + if ( + captured.get("config_sha256") != hashlib.sha256(config_bytes).hexdigest() + or captured.get("role") != role + or captured.get("instance_id") != cfg["instance_id"] + or captured.get("machine_id") != cfg["machine_id"] + ): + raise Refused("captured_result_config_binding") + sources = json.loads(Path(base["files"]["sources"]["path"]).read_bytes()) + if ( + fsha(base["files"]["sources"]["path"]) != base["files"]["sources"]["sha256"] + or sources != cfg["sources"] + ): + raise Refused("sources_map_mismatch") + bundle = Path(base["files"]["bundle"]["path"]) + if fsha(bundle) != base["files"]["bundle"]["sha256"]: + raise Refused("bundle_hash") + partial = Path(f"{root}.partial-{os.getpid()}") + partial.mkdir(mode=0o700) + try: + with tarfile.open(bundle) as source: + members = source.getmembers() + if len({m.name for m in members}) != len(members) or sorted( + m.name for m in members + ) != sorted(sources): + raise Refused("bundle_coverage_differs_from_signed_map") + for m in members: + put(source, m, partial / safe(m.name), sources[m.name], m.size) + # dedup: a repeated content travels once, under its first carried path. + by_sha = {files[n]["sha256"]: n for n in sorted(carried, reverse=True)} + with tarfile.open(archive) as source: + for name, want in sorted(files.items()): + if name not in carried and (hashes_only or want["sha256"] not in by_sha): + # hash-only rescue: proven by custody.json; dedup: content held locally + # (known input), bytes not carried. Hash is still in the manifest. + continue + carrier = name if name in carried else by_sha[want["sha256"]] + put( + source, + source.getmember("out/" + carrier), + partial / "out" / safe(name), + want["sha256"], + want["bytes"], + ) + copy( + Path(base["files"]["profile"]["path"]), partial / "profile.json", cfg["profile_sha256"] + ) + copy( + Path(base["files"]["registry"]["path"]), + partial / "registry.json", + cfg["image_digest"].removeprefix("sha256:"), + ) + copy( + run / f"cli-receipt-{role}.json", + partial / "receipts.json", + cfg["lifecycle_receipts_sha256"], + ) + durable_write(partial / "qualification.json", config_bytes) + for path in partial.rglob("*"): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise Refused("non_regular_output " + str(path)) + partial.rename(root) + except BaseException: + if partial.is_dir() and not partial.is_symlink(): + shutil.rmtree(partial) # only this invocation's own partial tree + raise + try: + nq.check_inputs(nq.Qualification.model_validate_json(config_bytes)) + except BaseException as exc: + shutil.rmtree(root) # the root this call just published; nothing else existed there + failure = { + "status": "CHECK_INPUTS_REJECTED_ROOT_REMOVED", + "role": role, + "error": f"{type(exc).__name__}: {exc}", + "rescue_tar_sha256": rescue["tar_sha256"], + "config_sha256": hashlib.sha256(config_bytes).hexdigest(), + } + durable_write( + run / f"materialize-failed-{role}.json", json.dumps(failure, sort_keys=True).encode() + ) + raise + receipt = { + "status": "MATERIALIZED_CHECK_INPUTS_OK", + "role": role, + "root": str(root), + "config_sha256": hashlib.sha256(config_bytes).hexdigest(), + "rescue_tar": str(archive), + "rescue_tar_sha256": rescue["tar_sha256"], + "rescue_generation": rescue["generation"], + "qualification_done": "present" if accepted is not None else "absent", + "qualification_result_sha256": result_sha, + "rescue_manifest_sha256": hashlib.sha256( + json.dumps(files, sort_keys=True).encode() + ).hexdigest(), + "rescue_files": len(files), + "rescue_bytes": sum(v["bytes"] for v in files.values()), + "bundle_sha256": base["files"]["bundle"]["sha256"], + "sources_sha256": base["files"]["sources"]["sha256"], + "source_files": len(sources), + "profile_sha256": cfg["profile_sha256"], + "registry_sha256": cfg["image_digest"].removeprefix("sha256:"), + "receipts_sha256": cfg["lifecycle_receipts_sha256"], + } + durable_write(run / f"materialized-{role}.json", json.dumps(receipt, sort_keys=True).encode()) + return receipt + + +def main() -> int: + run, base = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_bytes())["record"] + rc = 0 + for role in sys.argv[3:]: + try: + out = materialize(run, base, role) + print( + "MATERIALIZED", + role, + fsha(run / f"materialized-{role}.json"), + json.dumps(out, sort_keys=True), + flush=True, + ) + except ( + Refused, + orch.Reject, + nq.Reject, + OSError, + tarfile.TarError, + KeyError, + ValueError, + ) as exc: + print("MATERIALIZE_REJECT", role, type(exc).__name__, exc, flush=True) + rc = 1 + return rc + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/hypertrain/auditor/island_bisect.py b/src/hypertrain/auditor/island_bisect.py index be8cb46..4429e01 100644 --- a/src/hypertrain/auditor/island_bisect.py +++ b/src/hypertrain/auditor/island_bisect.py @@ -9,7 +9,7 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass from pathlib import Path from typing import Literal, assert_never @@ -108,8 +108,22 @@ def trace( return result, capture +TRACE_BYTES = 64 * 1024 * 1024 # per rank and per J-window trace file +Retrace = Callable[[int], Sequence[Sequence[TraceEntry]]] + + +def _skeleton(entries: Sequence[TraceEntry], base: int) -> list[tuple[object, ...]]: + # Shapes excluded: MoE dispatch rows are data-dependent; the event sequence is not. + return [(e.rank, e.step - base, e.microbatch, e.layer, e.op, e.dtype) for e in entries] + + class IslandParty: - """Read-only, rank-major transcript from authenticated execution artifacts.""" + """Read-only, rank-major transcript from authenticated execution artifacts. + + Commit-path traces cover window 1 only (the determinism probe). Any other window's hooks + come from ``retrace(leaf)``: a re-run of that window from checkpoint (leaf-1)*J. Hook events + per step do not depend on the step, so every window has window 1's skeleton (checked). + """ def __init__( self, @@ -117,12 +131,15 @@ def __init__( captures: Sequence[IslandCapture], *, J: int, + retrace: Retrace | None = None, ) -> None: - self.hotkey, self.J = hotkey, J + self.hotkey, self.J, self.retrace = hotkey, J, retrace + self._lazy: dict[int, tuple[TraceEntry, ...]] = {} if not captures or J < 1: raise RefereeError("missing island captures or invalid checkpoint stride") self.states = dict(captures[0].states) - if not self.states or any(c.states != self.states for c in captures): + # Rank 0 alone carries checkpoints (ranks are bitwise identical by validate_artifacts). + if not self.states or any(c.states and c.states != self.states for c in captures): raise RefereeError("rank checkpoint disagreement") self.entries = tuple( entry @@ -140,24 +157,65 @@ def _rank(entry: TraceEntry, rank: int) -> bool: raise RefereeError("trace rank binding mismatch") return True + @staticmethod + def read_trace(directory: Path, relative: str) -> list[TraceEntry]: + raw = confined(directory, relative).read_bytes() + if len(raw) > TRACE_BYTES: + raise RefereeError("local trace exceeds artifact byte budget") + # Local hooks contain -1 collective contexts, unlike unsigned wire envelopes. + return TypeAdapter(list[TraceEntry]).validate_json(raw) + @classmethod - def published(cls, hotkey: str, job: IslandJobV1, directory: Path) -> IslandParty: + def published( + cls, + hotkey: str, + job: IslandJobV1, + directory: Path, + retrace: Retrace | None = None, + ) -> IslandParty: + """``retrace`` defaults to launch_window on the publication's own backend.""" validate_artifacts(job, directory) + inner = job.manifest.training.inner captures: list[IslandCapture] = [] - for rank in range(job.manifest.training.reference_spec.layout.n_gpus): + n_gpus = job.manifest.training.reference_spec.layout.n_gpus + for rank in range(n_gpus): capture = IslandCapture() - raw = confined(directory, f"rank-{rank}/trace.json").read_bytes() - if len(raw) > 64 * 1024 * 1024: - raise RefereeError("local trace exceeds artifact byte budget") - # Local hooks contain -1 collective contexts, unlike unsigned wire envelopes. - capture.entries = TypeAdapter(list[TraceEntry]).validate_json(raw) - for t in range(0, job.manifest.training.inner.H + 1, job.manifest.training.inner.J): - capture.states[t] = confined( - directory, - f"rank-{rank}/checkpoints/{t}.safetensors", - ).read_bytes() + capture.entries = cls.read_trace(directory, f"rank-{rank}/trace.json") captures.append(capture) - return cls(hotkey, captures, J=job.manifest.training.inner.J) + for t in range(0, inner.H + 1, inner.J): + captures[0].states[t] = confined( + directory, f"rank-0/checkpoints/{t}.safetensors" + ).read_bytes() + if retrace is None: + + def retrace(leaf: int) -> list[list[TraceEntry]]: + from hypertrain.miner.island_launch import launch_window + + launch_window(job, directory, leaf) + return [ + cls.read_trace(directory, f"rank-{rank}/windows/{leaf}.json") + for rank in range(n_gpus) + ] + + return cls(hotkey, captures, J=inner.J, retrace=retrace) + + def _entries(self, leaf: int) -> tuple[TraceEntry, ...]: + lo, hi = (leaf - 1) * self.J, leaf * self.J + got = tuple(e for e in self.entries if lo < e.step <= hi) + if got or self.retrace is None or leaf == 1: + return got + if leaf not in self._lazy: + ranks = self.retrace(leaf) + lazy = tuple( + e + for rank, entries in enumerate(ranks) + for e in entries + if self._rank(e, rank) and lo < e.step <= hi + ) + if _skeleton(lazy, lo) != _skeleton(self._entries(1), 0): + raise RefereeError("re-run window trace differs from the probe window shape") + self._lazy[leaf] = lazy + return self._lazy[leaf] def state_blob(self, leaf: int) -> bytes: try: @@ -174,7 +232,7 @@ def state_root(self, leaf: int) -> str: def window(self, leaf: int) -> tuple[tuple[TraceEntry, ...], tuple[int, ...], list[str]]: if not 1 <= leaf <= self.H // self.J: raise RefereeError("window outside committed trajectory") - entries = tuple(e for e in self.entries if (leaf - 1) * self.J < e.step <= leaf * self.J) + entries = self._entries(leaf) if not entries: raise RefereeError("missing actual island hooks") chain = [self.state_root(leaf - 1)] @@ -202,17 +260,30 @@ def span(self, level: Level, ctx: tuple[int, ...]) -> int: case "layer": if len(ctx) != 1: raise RefereeError("LAYER context requires window") - return len(self.window(ctx[0])[1]) + return len(self._shape(ctx[0])) case "op": if len(ctx) != 2: raise RefereeError("OP context requires window/span") - _, ends, _ = self.window(ctx[0]) + ends = self._shape(ctx[0]) if not 0 <= ctx[1] < len(ends): raise RefereeError("unknown layer span") return ends[ctx[1]] - (ends[ctx[1] - 1] if ctx[1] else 0) case other: assert_never(other) + def _shape(self, leaf: int) -> tuple[int, ...]: + """Span ends of a window, from an already traced one (all windows share the skeleton), + so sizing a dispute never forces a re-run.""" + if not 1 <= leaf <= self.H // self.J: + raise RefereeError("window outside committed trajectory") + known = ( + leaf + if leaf in self._lazy + or any((leaf - 1) * self.J < e.step <= leaf * self.J for e in self.entries) + else 1 + ) + return self.window(known)[1] + def hashes(self, level: Level, ctx: tuple[int, ...], at: Sequence[int]) -> list[str]: span = self.span(level, ctx) if any(not 0 <= i <= span for i in at): diff --git a/src/hypertrain/challenge/store.py b/src/hypertrain/challenge/store.py index fe13dc9..58e4984 100644 --- a/src/hypertrain/challenge/store.py +++ b/src/hypertrain/challenge/store.py @@ -6119,7 +6119,7 @@ def schedule_audits_v2(self, run_id: str, w: int) -> list[str]: missing.append((entry, identity, job_id, commit)) if active + len(missing) > policy.max_queued_jobs: raise ChallengeError(503, "audit capacity unavailable; no new reservation") - absolute = min(body.d_final, body.d_audit + 200) + absolute = min(body.d_final, body.d_audit + policy.absolute_rounds()) if absolute <= now + 1: raise ChallengeError(503, "audit absolute horizon exhausted") result = [] @@ -6309,7 +6309,10 @@ def lease_v2(self, run_id: str, raw: bytes) -> dict[str, Any] | None: if row is None: return None nonce = secrets.token_hex(32) - expires = min(now + 100, row["absolute"]) + lease_rounds = AuditPolicyV2.model_validate_json( + self.objects.get(manifest.network.audit_policy_hash) + ).lease_rounds + expires = min(now + lease_rounds, row["absolute"]) identity = f"{row['w']}:{row['hotkey']}" start = StartStateV2.model_validate(self._record_v2(run_id, "start", identity)) commit_env = self._record_v2(run_id, "commit", identity) @@ -6567,7 +6570,11 @@ def read(key: str) -> bytes: directory = self.state_dir / "jobs-v2" / str(w) / hotkey directory.mkdir(parents=True, exist_ok=True) state = get_large(start.state_object_sha256, self.objects.get) - if manifest.training.inner.state_policy != "carry": + # carry: also stage the public theta-only object; else the start state is theta only. + theta_only: bytes | None = None + if manifest.training.inner.state_policy == "carry": + theta_only = pack_state(unpack_state(state)[0]) + else: state = pack_state(unpack_state(state)[0]) inputs = { "start_state": state, @@ -6602,7 +6609,10 @@ def read(key: str) -> bytes: deadline=manifest.training.beacon.genesis_time + (int(body["d_commit"]) - 1) * 3, ) self._put_record_v2(run_id, "island-job", f"{w}:{hotkey}", job.body()) - return {"job": job.body(), "objects": hashes} + staged: dict[str, Any] = {"job": job.body(), "objects": hashes} + if theta_only is not None: # a miner rebuilds start_state from it + its own m/v + staged["theta"] = put_large(theta_only, self.objects.put) + return staged def execute_audit_v2(self, run_id: str, job_id: str, raw: bytes) -> dict[str, Any]: """Independent service execution; auditor signature cannot manufacture MATCH.""" diff --git a/src/hypertrain/data/store.py b/src/hypertrain/data/store.py index e8201dd..95b594f 100644 --- a/src/hypertrain/data/store.py +++ b/src/hypertrain/data/store.py @@ -17,7 +17,7 @@ import tempfile from collections import deque from collections.abc import Callable, Iterator, Mapping -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor from itertools import islice from pathlib import Path from typing import Protocol @@ -309,31 +309,47 @@ def parse_index(raw: bytes) -> PartIndexV1 | None: return index -def _publish(chunks: Callable[[], bytes], put: Put) -> str: +def _publish(chunks: Callable[[], bytes], put: Put, workers: int = 1) -> str: + """workers > 1 PUTs parts concurrently (at most 2 * workers parts = 2 * workers MiB in + flight, so `put` must be thread-safe); the index is still PUT last, after every part.""" buf, nxt = chunks(), chunks() if not nxt: if buf.startswith(INDEX_MAGIC): raise StoreError("plain object must not begin with the part-index magic") return _put(put, buf) - whole, size, parts = hashlib.sha256(), 0, list[str]() - while buf: - if len(parts) >= MAX_PARTS: - raise StoreError("object exceeds MAX_PARTS * PART_BYTES") - if nxt and len(buf) != PART_BYTES: - raise StoreError("short read before end of object") - whole.update(buf) - size += len(buf) - parts.append(_put(put, buf)) - buf, nxt = nxt, (chunks() if nxt else b"") + whole, size, parts, count = hashlib.sha256(), 0, list[str](), 0 + pool = ThreadPoolExecutor(workers) if workers > 1 else None + inflight: deque[Future[str]] = deque() + try: + while buf: + if count >= MAX_PARTS: + raise StoreError("object exceeds MAX_PARTS * PART_BYTES") + if nxt and len(buf) != PART_BYTES: + raise StoreError("short read before end of object") + whole.update(buf) + size += len(buf) + count += 1 + if pool is None: + parts.append(_put(put, buf)) + else: + if len(inflight) >= 2 * workers: + parts.append(inflight.popleft().result()) + inflight.append(pool.submit(_put, put, buf)) + buf, nxt = nxt, (chunks() if nxt else b"") + while inflight: + parts.append(inflight.popleft().result()) + finally: # error/cancel: drop queued PUTs, wait for running ones, never publish the index + if pool is not None: + pool.shutdown(wait=True, cancel_futures=True) index = PartIndexV1(sha256=whole.hexdigest(), size=size, parts=parts) return _put(put, index.encode()) # last: readers never see an index before its parts -def put_file(path: Path, put: Put) -> str: +def put_file(path: Path, put: Put, workers: int = 1) -> str: """Store `path`; returns the key of the plain object (<= PART_BYTES) or of its part index. The final put() call's bytes are the referenced object (len <= MAX_MANIFEST_BYTES).""" with Path(path).open("rb") as f: - return _publish(lambda: f.read(PART_BYTES), put) + return _publish(lambda: f.read(PART_BYTES), put, workers) def put_large(data: bytes, put: Put) -> str: diff --git a/src/hypertrain/datasets/build.py b/src/hypertrain/datasets/build.py index 8c2e59d..505098b 100644 --- a/src/hypertrain/datasets/build.py +++ b/src/hypertrain/datasets/build.py @@ -6,7 +6,9 @@ from __future__ import annotations +import gzip import hashlib +import io import json from collections.abc import Callable, Iterable, Iterator from pathlib import Path @@ -61,11 +63,23 @@ def parquet_loader(cache: Path) -> Loader: def load(src: Source) -> Iterable[Row]: import pyarrow.parquet as pq - files = sorted((cache / src.id).rglob("*.parquet")) + root = cache / src.id + files = sorted(p for g in ("*.parquet", "*.jsonl.zst", "*.json.gz") for p in root.rglob(g)) if not files: - raise BuildError(f"{src.id}: no parquet files in {cache / src.id}") + raise BuildError(f"{src.id}: no parquet/jsonl.zst/json.gz files in {root}") for p in files: - yield from pq.read_table(p).to_pylist() + if p.suffix == ".parquet": + yield from pq.read_table(p).to_pylist() + continue + # dclm-baseline ships *.jsonl.zst and peS2o v2 ships *.json.gz at the pinned revisions + if p.suffix == ".zst": + import zstandard + + raw = io.TextIOWrapper(zstandard.ZstdDecompressor().stream_reader(p.open("rb"))) + else: + raw = io.TextIOWrapper(gzip.open(p, "rb")) + with raw: + yield from (json.loads(line) for line in raw if line.strip()) return load diff --git a/src/hypertrain/datasets/registry.json b/src/hypertrain/datasets/registry.json index 8ff5a31..0bfae30 100644 --- a/src/hypertrain/datasets/registry.json +++ b/src/hypertrain/datasets/registry.json @@ -215,7 +215,7 @@ "dclm-baseline": { "config": null, "files": [ - "*.parquet" + "**/*.jsonl.zst" ], "licence": "CC-BY-4.0", "repo": "mlfoundations/dclm-baseline-1.0", @@ -320,7 +320,7 @@ "pes2o-v2": { "config": "v2", "files": [ - "data/v2/*.parquet" + "data/v2/*.json.gz" ], "licence": "ODC-By-1.0", "repo": "allenai/peS2o", diff --git a/src/hypertrain/gpu_ops/network_qualification.py b/src/hypertrain/gpu_ops/network_qualification.py index 272fc13..2ff31f1 100644 --- a/src/hypertrain/gpu_ops/network_qualification.py +++ b/src/hypertrain/gpu_ops/network_qualification.py @@ -151,7 +151,7 @@ class Result(Boundary): def round_files(ranks: int, H: int, J: int) -> list[str]: # noqa: N803 - manifest names - """Every rank's final artifacts plus each J-th checkpoint of an H-step round.""" + """Every rank's final artifacts plus rank-0's J-th checkpoints (only rank 0 writes them).""" if min(ranks, H, J) < 1 or H % J: raise Reject("qualification_layout_invalid") return [ @@ -163,7 +163,7 @@ def round_files(ranks: int, H: int, J: int) -> list[str]: # noqa: N803 - manife "delta.bin", "leaves.json", "trace.json", - *(f"checkpoints/{t}.safetensors" for t in range(0, H + 1, J)), + *(f"checkpoints/{t}.safetensors" for t in range(0, H + 1, J) if not rank), ) ] diff --git a/src/hypertrain/miner/cli.py b/src/hypertrain/miner/cli.py index 15c8672..f13fb3e 100644 --- a/src/hypertrain/miner/cli.py +++ b/src/hypertrain/miner/cli.py @@ -22,17 +22,87 @@ import numpy.typing as npt from hypertrain.data.shards import ShardSamples, ShardSetManifest, verify_shards -from hypertrain.datasets.shards16 import ShardSet16Manifest, U16ShardSamples, verify_shards16 +from hypertrain.datasets.fetch import prefetch +from hypertrain.datasets.shards16 import ( + U16, + ShardSet16Manifest, + U16ShardSamples, + shard_name16, + verify_shards16, +) from hypertrain.miner.core import Miner, MinerConfig, MinerError, NetworkMiner, canonical from hypertrain.miner.dispute_watch import add_watch_args, run_follow from hypertrain.miner.keys import create_key, list_keys, revoke_key from hypertrain.miner.update import Updater, UpdateRejected +from hypertrain.protocol.hashing import MerkleTree from hypertrain.protocol.messages import RunManifest from hypertrain.trainer.loop import SampleFn +DATA_BASES_ENV = "HYPERTRAIN_DATA_BASES" +DEFAULT_DATA_BASE = "http://185.8.107.43:18980/" +MAX_DATASET_MANIFEST_BYTES = 64 << 20 + + +def _data_bases() -> list[str]: + raw = os.environ.get(DATA_BASES_ENV) or DEFAULT_DATA_BASE + bases = [b.strip() for b in raw.split(",") if b.strip()] + if not bases or any(not b.startswith(("http://", "https://")) for b in bases): + raise MinerError(f"{DATA_BASES_ENV} must be comma-separated http(s) base URLs") + return bases + + +def _dataset_manifest(data_dir: Path, bases: list[str]) -> ShardSet16Manifest: + """Local manifest.json, else fetched (bounded) from the first base that serves it.""" + path = data_dir / "manifest.json" + if not path.is_file(): + errors = [] + for base in bases: + try: + with httpx.stream("GET", base.rstrip("/") + "/manifest.json", timeout=60) as r: + r.raise_for_status() + body = bytearray() + for chunk in r.iter_bytes(): + body += chunk + if len(body) > MAX_DATASET_MANIFEST_BYTES: + raise MinerError("dataset manifest exceeds 64 MiB") + except (httpx.HTTPError, MinerError) as e: + errors.append(f"{base}: {e}") + continue + data_dir.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + tmp.write_bytes(body) + tmp.replace(path) + break + else: + raise MinerError(f"dataset manifest unavailable: {errors[-3:]}") + return ShardSet16Manifest.from_json(path.read_text()) + + +class _RemoteUnits: + """Samples from sha256-verified units fetched by HTTP Range on first use, cached under + data_dir/units/. ponytail: on-demand per unit (8 units per H=512 slot); add a + whole-assignment prefetch once the miner core passes its sample ids ahead of training.""" + + def __init__(self, data_dir: Path, m: ShardSet16Manifest, bases: list[str]) -> None: + self.m, self.bases, self.cache = m, bases, data_dir / "units" + self.maps: dict[int, npt.NDArray[np.uint16]] = {} + + def row(self, i: int) -> npt.NDArray[np.uint32]: + if not 0 <= i < self.m.n_samples: + raise IndexError(i) + u, k = divmod(i, self.m.unit) + if u not in self.maps: + for _ in prefetch(self.m, [u], self.cache, self.bases, streams=1, depth=1): + pass + path = self.cache / self.m.unit_sha256s[u] + self.maps[u] = np.memmap(path, dtype=U16, mode="r").reshape(-1, self.m.seq_len + 1) + return self.maps[u][k].astype(np.uint32) + def _u16_sampler(data_dir: Path, manifest: RunManifest) -> SampleFn: - m = ShardSet16Manifest.from_json((data_dir / "manifest.json").read_text()) + local = (data_dir / shard_name16(0)).is_file() + bases = [] if local else _data_bases() + m = _dataset_manifest(data_dir, bases) ds = manifest.dataset if (m.merkle_root, m.shard_sha256_root, m.n_samples, m.unit_sha256_root) != ( ds.merkle_root, @@ -43,6 +113,19 @@ def _u16_sampler(data_dir: Path, manifest: RunManifest) -> SampleFn: raise MinerError( "local shards are not the run's dataset (merkle/shard root/n_samples/unit root)" ) + if not local: + # Untrusted manifest bytes: the unit list must hash to the signed root, and the units + # must tile n_samples exactly; each fetched unit is then checked against its listed sha. + if ( + len(m.unit_sha256s) * m.unit != m.n_samples + or m.samples_per_shard % m.unit + or MerkleTree([bytes.fromhex(h) for h in m.unit_sha256s]).root.hex() + != ds.unit_sha256_root + or MerkleTree([bytes.fromhex(h) for h in m.shard_sha256s]).root.hex() + != ds.shard_sha256_root + ): + raise MinerError("remote dataset manifest unit/shard lists differ from signed roots") + return _RemoteUnits(data_dir, m, bases).row errors = verify_shards16(data_dir, m) if errors: raise MinerError(f"shard verification failed: {errors[:3]}") diff --git a/src/hypertrain/miner/core.py b/src/hypertrain/miner/core.py index ae3afa2..98f4e69 100644 --- a/src/hypertrain/miner/core.py +++ b/src/hypertrain/miner/core.py @@ -15,6 +15,7 @@ import os import shutil import stat +import tempfile import threading import time import tomllib @@ -29,7 +30,7 @@ if TYPE_CHECKING: from hypertrain.gpu_ops.work_screen import IslandLaunch from hypertrain.miner.island_launch import IslandArtifacts - from hypertrain.protocol.messages_v2 import IslandJobV1 + from hypertrain.protocol.messages_v2 import ArtifactRef, IslandJobV1 import hypertrain.trainer # noqa: F401 (determinism setup before torch) from hypertrain.auditor.bisect import Executor @@ -53,6 +54,7 @@ # transport error or a 5xx; the last failure propagates. ~31 s worst case per request. RETRY_DELAYS: tuple[float, ...] = (1, 2, 4, 8, 16) FETCH_WORKERS = 16 # concurrent 1 MiB part GETs per job input (bounded window, ordered sink) +UPLOAD_WORKERS = 16 # concurrent 1 MiB part PUTs per artifact (bounded window, index last) def _retrying[T](what: str, fn: Callable[[], T]) -> T: @@ -73,10 +75,66 @@ def _retrying[T](what: str, fn: Callable[[], T]) -> T: raise AssertionError("unreachable") +def _upload_ref(path: Path, put: Callable[[bytes], str]) -> ArtifactRef: + """artifact_ref(path, put) with UPLOAD_WORKERS concurrent part PUTs (index still last).""" + from hypertrain.data.store import put_file + from hypertrain.protocol.messages_v2 import ArtifactRef + + last = b"" + + def record(data: bytes) -> str: + nonlocal last + last = data # the index (or plain object) is put after every part has completed + return put(data) + + sha = put_file(path, record, UPLOAD_WORKERS) + return ArtifactRef(sha256=sha, size=len(last)) + + class MinerError(RuntimeError): pass +def _st_header(f: Any) -> tuple[int, dict[str, Any]]: + """safetensors header: (data start offset, {name: {dtype, shape, data_offsets}}).""" + n = int.from_bytes(f.read(8), "little") + if not 0 < n <= 100 << 20: + raise ValueError("bad safetensors header length") + header = json.loads(f.read(n)) + header.pop("__metadata__", None) + return 8 + n, header + + +def _overwrite_theta(state: Path, theta: Path, chunk: int = 16 << 20) -> bool: + """Overwrite state's theta/* tensors in place with theta's, chunk by chunk. + + False when the theta file's tensors (names/dtypes/shapes) differ from state's theta/*.""" + with state.open("r+b") as dst, theta.open("rb") as src: + dst_base, dst_header = _st_header(dst) + src_base, src_header = _st_header(src) + if set(src_header) != {k for k in dst_header if k.startswith("theta/")}: + return False + for key, entry in src_header.items(): + target = dst_header[key] + if (entry["dtype"], entry["shape"]) != (target["dtype"], target["shape"]): + return False + (a, b), (c, d) = entry["data_offsets"], target["data_offsets"] + if b - a != d - c: + return False + src.seek(src_base + a) + dst.seek(dst_base + c) + left = b - a + while left: + data = src.read(min(chunk, left)) + if not data: + raise ValueError("theta file truncated") + dst.write(data) + left -= len(data) + dst.flush() + os.fsync(dst.fileno()) + return True + + class KeyfileError(MinerError): pass @@ -816,9 +874,8 @@ def traced_launch( self.kp, "WorkScreenV2", self.run_id, screen, challenge.deadline_beacon ), } - from hypertrain.gpu_ops.work_screen import artifact_ref - put = self._bounded_put(job.deadline, cancel) + # Eager: admission.proof verifies every proof ref against stored bytes (one-time join). for ref, filename in zip( proof.artifact_refs, ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"), @@ -827,7 +884,7 @@ def traced_launch( if time.time() >= job.deadline or cancel is not None and cancel.is_set(): raise MinerError("probe deadline/cancellation") path = job_path.parent / "published" / "rank-0" / filename - if artifact_ref(path, put) != ref: + if _upload_ref(path, put) != ref: raise MinerError("published artifact differs from screened ref") if time.time() >= job.deadline or cancel is not None and cancel.is_set(): raise MinerError("probe deadline/cancellation") @@ -867,10 +924,19 @@ def put() -> str: return _retrying("PUT object " + digest, put) - def _fetch_inputs(self, job: IslandJobV1, objects: Mapping[str, str], directory: Path) -> None: + def _fetch_inputs( + self, + job: IslandJobV1, + objects: Mapping[str, str], + directory: Path, + theta: str | None = None, + previous: Path | None = None, + ) -> None: """objects: name -> stored ref (plain or ht-parts-v1 index); the job pins whole SHAs. - Each file is verified and atomically replaced; a bad ref never leaves a file behind.""" + Each file is verified and atomically replaced; a bad ref never leaves a file behind. + previous: this miner's own prior publication (rank-0); ef_in, and start_state rebuilt + from the public `theta` ref + its own m/v/step, are reused when they hash to the pins.""" from hypertrain.data.store import StoreError, fetch_to whole = { @@ -884,6 +950,8 @@ def _fetch_inputs(self, job: IslandJobV1, objects: Mapping[str, str], directory: raise MinerError("job object names differ from the job's object paths") for name, digest in objects.items(): dest = directory / job.object_paths[name] + if name in whole and self._local_input(name, whole[name], dest, theta, previous): + continue try: fetch_to(digest, self._get_object, dest, workers=FETCH_WORKERS) except StoreError as e: @@ -894,6 +962,45 @@ def _fetch_inputs(self, job: IslandJobV1, objects: Mapping[str, str], directory: dest.unlink() raise MinerError("job input hash mismatch") + def _local_input( + self, name: str, whole: str, dest: Path, theta: str | None, previous: Path | None + ) -> bool: + """True iff dest now holds bytes rebuilt locally that hash to `whole` (else untouched). + + Streamed on disk (bounded RAM): start_state = the previous state file with its theta/* + byte ranges overwritten from the fetched theta-only file (same names/shapes/dtypes, so + the safetensors header and offsets are identical). Any mismatch, or a corrupt or + unfetchable theta ref, returns False so the caller downloads the full pinned object.""" + from hypertrain.data.store import StoreError, fetch_to + + if previous is None: + return False + source = previous / ("ef.safetensors" if name == "ef_in" else "state.safetensors") + if name not in ("ef_in", "start_state") or not source.is_file(): + return False + if name == "start_state" and theta is None: + return False + fd, tmp = tempfile.mkstemp(dir=dest.parent, prefix=f".{dest.name}.", suffix=".part") + os.close(fd) + out, theta_tmp = Path(tmp), dest.parent / f".{dest.name}.theta.part" + try: + shutil.copyfile(source, out) + if name == "start_state" and theta is not None: + fetch_to(theta, self._get_object, theta_tmp, workers=FETCH_WORKERS) + if not _overwrite_theta(out, theta_tmp): + return False + with out.open("rb") as f: + if hashlib.file_digest(f, "sha256").hexdigest() != whole: + return False + os.replace(out, dest) + return True + except (StoreError, MinerError, ValueError, httpx.HTTPError) as e: + log.warning("local %s rebuild failed (%s); downloading the pinned object", name, e) + return False + finally: + out.unlink(missing_ok=True) + theta_tmp.unlink(missing_ok=True) + def _get_object(self, digest: str) -> bytes: """Bounded streamed GET: ht-parts-v1 parts and indexes are <= 1 MiB each, so a larger body is refused before it is buffered (legacy >1 MiB plain objects are not fetchable).""" @@ -1023,7 +1130,13 @@ def run_round(self, w: int) -> str: job = IslandJobV1.model_validate(staged["job"]) directory = self.directory / str(w) directory.mkdir(parents=True, exist_ok=True) - self._fetch_inputs(job, staged["objects"], directory) + self._fetch_inputs( + job, + staged["objects"], + directory, + staged.get("theta"), + self.directory / str(w - 1) / "published" / "rank-0" if w else None, + ) artifacts = launch_island( job, directory, @@ -1063,10 +1176,11 @@ def run_round(self, w: int) -> str: put = self._bounded_put( self.manifest.training.beacon.genesis_time + (schedule.d_final - 1) * 3, None ) - refs = [ - artifact_ref(path, put) - for path in (artifacts.state, artifacts.ef, artifacts.delta, artifacts.leaves) - ] + # state/ef/delta are committed by ref only: no round verifier reads them from the store + # (leaves_v2 reads refs[-1]; the delta reaches the master via relay retrieval; audits + # replay from master-staged inputs; disputes serve local checkpoints). + refs = [artifact_ref(path) for path in (artifacts.state, artifacts.ef, artifacts.delta)] + refs.append(_upload_ref(artifacts.leaves, put)) descriptor = WorkProof( admission_id=status["record"]["admission_id"], challenge_hash=sha256_hex(canonical(commit)), diff --git a/src/hypertrain/miner/island_launch.py b/src/hypertrain/miner/island_launch.py index b20bd66..c290f00 100644 --- a/src/hypertrain/miner/island_launch.py +++ b/src/hypertrain/miner/island_launch.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import hashlib import json import os @@ -484,8 +485,10 @@ def validate_artifacts(job: IslandJobV1, directory: Path) -> IslandArtifacts: job.run_id, job.w, p.t, -1 ): raise IslandFailure("leaf assignment/rng mismatch") + if rank: + continue # checkpoints live under rank-0 only; rank-N leaves equal rank-0's chk, opt = unpack_state( - confined(directory, f"rank-{rank}/checkpoints/{p.t}.safetensors").read_bytes() + confined(directory, f"rank-0/checkpoints/{p.t}.safetensors").read_bytes() ) from hypertrain.trainer.loop import stage_states @@ -636,6 +639,49 @@ def launch_island( return validate_artifacts(job, published) +def launch_window(job: IslandJobV1, published: Path, leaf: int, *, timeout: float = 3600.0) -> None: + """Lazy dispute trace: re-run committed window ``leaf`` from rank-0's checkpoint (leaf-1)*J + with hooks on, writing rank-N/windows/.json. The worker refuses a re-run whose leaf + differs from the committed one. Cached: a finished window is never re-run.""" + published = published.resolve() + n_gpus = job.manifest.training.reference_spec.layout.n_gpus + outputs = [published / f"rank-{r}/windows/{leaf}.json" for r in range(n_gpus)] + summary = load_json(confined(published, "rank-0/summary.json").read_bytes()) + if not isinstance(summary, dict) or summary.get("backend") not in ("cpu", "cuda"): + raise IslandFailure("publication backend missing") + ref = job.manifest.training.reference_spec + env = dict( + os.environ, + CUBLAS_WORKSPACE_CONFIG=ref.env.CUBLAS_WORKSPACE_CONFIG, + OMP_NUM_THREADS="1", + CUDA_DISABLE_PTX_JIT=ref.env.CUDA_DISABLE_PTX_JIT, + HT_ISLAND_TRACE="1", + HT_ISLAND_WINDOW=str(leaf), + ) + with flock(published.parent / "launch.lock"): + if all(p.is_file() for p in outputs): + return + proc = subprocess.Popen( # noqa: S603 - fixed interpreter/package, typed layout + launch_argv(job, published, summary["backend"]), + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + start_new_session=True, + ) + try: + _, err = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired as e: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + raise IslandFailure("window re-run deadline expired") from e + finally: + with contextlib.suppress(ProcessLookupError): + os.killpg(proc.pid, signal.SIGKILL) + if proc.returncode != 0 or not all(p.is_file() for p in outputs): + tail = err[-4000:].decode(errors="replace") + raise IslandFailure(f"window re-run failure {proc.returncode}: {tail}") + + def _launch_unbounded( job: IslandJobV1, attempt: Path, diff --git a/src/hypertrain/miner/island_worker.py b/src/hypertrain/miner/island_worker.py index 17dfb0f..80df719 100644 --- a/src/hypertrain/miner/island_worker.py +++ b/src/hypertrain/miner/island_worker.py @@ -123,10 +123,62 @@ def move(p: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]: def get(i: int) -> npt.NDArray[np.uint32]: return samples[i] + traces: list[dict[str, object]] = [] + + def trace(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: + raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() + traces.append( + { + "rank": ctx.rank, + "step": ctx.step, + "microbatch": ctx.microbatch, + "layer": ctx.layer, + "op": ctx.op, + "shape": list(x.shape), + "dtype": str(x.dtype), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ) + return x + + window = os.environ.get("HT_ISLAND_WINDOW") + if window is not None: + # Lazy dispute trace: re-run committed window l from rank-0's checkpoint (l-1)*J. + leaf, J = int(window), cfg.inner.J + if not 1 <= leaf <= cfg.inner.H // J: + raise IslandFailure("window outside committed trajectory") + start, state = unpack_state( + confined(directory, f"rank-0/checkpoints/{(leaf - 1) * J}.safetensors").read_bytes() + ) + if state is None: + raise IslandFailure("checkpoint omits optimizer") + state.m, state.v = move(state.m), move(state.v) + result = train_island( + cfg, + lay, + DistComm(), + move(start), + Assignment(run_id, job.w, tuple(job.sample_ids), job.global_step0), + get, + carry=state, + hook=trace, + steps=((leaf - 1) * J, leaf * J), + ) + committed = json.loads( + confined(directory, f"rank-{dist.get_rank()}/leaves.json").read_bytes() + ) + if result.leaves[-1].preimage.model_dump(mode="json") != committed[leaf]: + raise IslandFailure("window re-run diverged from the committed leaf") + path = directory / f"rank-{dist.get_rank()}/windows/{leaf}.json" + path.parent.mkdir(exist_ok=True) + durable_write(path, canonicalize(traces)) + return + started = time.monotonic_ns() out = directory / f"rank-{dist.get_rank()}" out.mkdir() - (out / "checkpoints").mkdir() + if dist.get_rank() == 0: + (out / "checkpoints").mkdir() # Async checkpoint: train_island hands over freshly gathered tensors it never mutates, so # one writer thread serialises them while training continues; drained before outputs. @@ -134,6 +186,8 @@ def get(i: int) -> npt.NDArray[np.uint32]: saves: list[Future[None]] = [] def checkpoint(t: int, theta: dict[str, torch.Tensor], state: OptState) -> None: + if dist.get_rank(): + return # ranks are bitwise identical (validate_artifacts); rank 0 keeps the copy saves.append( saver.submit( lambda: durable_write( @@ -142,24 +196,6 @@ def checkpoint(t: int, theta: dict[str, torch.Tensor], state: OptState) -> None: ) ) - traces = [] - - def trace(ctx: TraceContext, x: torch.Tensor) -> torch.Tensor: - raw = x.detach().cpu().contiguous().reshape(-1).view(torch.uint8).numpy().tobytes() - traces.append( - { - "rank": ctx.rank, - "step": ctx.step, - "microbatch": ctx.microbatch, - "layer": ctx.layer, - "op": ctx.op, - "shape": list(x.shape), - "dtype": str(x.dtype), - "sha256": hashlib.sha256(raw).hexdigest(), - } - ) - return x - def deadline() -> None: if time.time() >= job.deadline: raise IslandFailure("job deadline expired") @@ -177,6 +213,9 @@ def deadline() -> None: hook=trace if os.environ.get("HT_ISLAND_TRACE") == "1" else None, cancel=deadline, checkpoint=checkpoint, + # Commit path: window 1 is the cross-rank/cross-host determinism probe; later + # windows are traced lazily on dispute (HT_ISLAND_WINDOW), so trace.json stays small. + hook_until=cfg.inner.J, ) for f in saves: f.result() # re-raises a failed write before any result is published diff --git a/src/hypertrain/protocol/messages_v2.py b/src/hypertrain/protocol/messages_v2.py index 2511b90..def64c3 100644 --- a/src/hypertrain/protocol/messages_v2.py +++ b/src/hypertrain/protocol/messages_v2.py @@ -409,10 +409,13 @@ def _bounds(self) -> AuditJobV2: raise ValueError("replay budget must cover H and not exceed 2H") if not self.created_beacon < self.lease_expires <= self.absolute_deadline: raise ValueError("invalid audit lease deadline") - if self.lease_expires > self.created_beacon + 100: - raise ValueError("lease exceeds 100 beacon rounds") - if self.absolute_deadline > self.created_beacon + 200: - raise ValueError("absolute audit deadline exceeds 200 beacon rounds") + # ponytail: the job carries no policy bytes, so this structural cap uses the protocol + # ceiling MAX_STEP_SECONDS_BOUND; the store issues the exact policy-derived lease. + lease = audit_lease_rounds(h, MAX_STEP_SECONDS_BOUND) + if self.lease_expires > self.created_beacon + lease: + raise ValueError(f"lease exceeds {lease} beacon rounds") + if self.absolute_deadline > self.created_beacon + 2 * lease: + raise ValueError(f"absolute audit deadline exceeds {2 * lease} beacon rounds") if len(self.sample_ids) != self.manifest.training.batch_samples(): raise ValueError("job assignment length mismatch") if self.start_state.run_id != self.run_id: @@ -617,8 +620,21 @@ class DisputePolicyV2(WireModel): max_referee_cost_units: U64 +BEACON_PERIOD_SECONDS = 3 +MAX_STEP_SECONDS_BOUND = 16.0 + + +def audit_lease_rounds(h: int, step_seconds_bound: float) -> int: + """Beacon rounds one audit lease lasts: a 2H replay at the step bound plus 100 rounds of + slack. H <= 30 keeps the original fixed 100-round lease (absolute cap = 2 * lease).""" + if h <= 30: + return 100 + return math.ceil(2 * h * step_seconds_bound / BEACON_PERIOD_SECONDS) + 100 + + class AuditPolicyV2(WireModel): - lease_rounds: Literal[100] + lease_rounds: Positive + step_seconds_bound: F32 = "00008040" # f32 4.0 s per untraced replay step max_attempts: Literal[2] max_concurrent_per_auditor: Literal[1] max_running_jobs: Literal[2] @@ -629,6 +645,17 @@ class AuditPolicyV2(WireModel): def validate_training(self, training: TrainingManifestV2) -> None: if self.max_steps_per_attempt != 2 * training.inner.H: raise ValueError("audit policy budget must equal 2H") + if self.lease_rounds != audit_lease_rounds(training.inner.H, self.step_bound()): + raise ValueError("audit lease must equal the H/step-bound derived lease") + + def step_bound(self) -> float: + bound = f32val(self.step_seconds_bound) + if not 0 < bound <= MAX_STEP_SECONDS_BOUND: # also rejects NaN + raise ValueError("audit step_seconds_bound must be in (0, 16] seconds") + return bound + + def absolute_rounds(self) -> int: + return 2 * self.lease_rounds class IslandJobV1(WireModel): diff --git a/src/hypertrain/trainer/island.py b/src/hypertrain/trainer/island.py index 3b57b4a..e2f2c4f 100644 --- a/src/hypertrain/trainer/island.py +++ b/src/hypertrain/trainer/island.py @@ -922,9 +922,17 @@ def train_island( cancel: CancelHook | None = None, after_step: Callable[[int, Params], None] | None = None, checkpoint: Callable[[int, Params, OptState], None] | None = None, + hook_until: int | None = None, + steps: tuple[int, int] | None = None, ) -> RoundResult: """One H-step round on this rank; every rank returns the same full RoundResult. + ``hook_until`` drops ``hook`` after that step (the commit path traces window 1 only). + ``steps=(t0, t1)`` re-runs just steps t0+1..t1 from the committed full state at t0, passed as + ``theta_start``/``carry`` (lazy per-window dispute trace); its leaves cover only that span + and its delta/EF are meaningless. Per-step hook events never depend on earlier steps, so a + re-run window emits exactly the events of the continuous round. + ``reduction="bucketed"`` swaps the gradient all_gather for the bucketed reduce-scatter (bitwise identical result, see _reduce_bucketed). ``"zero2"`` shards gradients onto the ZeRO-1 owners (_reduce_to_owner); ``"zero3"`` additionally keeps only owned parameters between @@ -950,6 +958,8 @@ def train_island( cancel, after_step, checkpoint, + hook_until, + steps, ) @@ -968,8 +978,16 @@ def _train_island( cancel: CancelHook | None = None, after_step: Callable[[int, Params], None] | None = None, checkpoint: Callable[[int, Params, OptState], None] | None = None, + hook_until: int | None = None, + steps: tuple[int, int] | None = None, ) -> RoundResult: require_threads(cfg.cpu_threads) + H, J = cfg.inner.H, cfg.inner.J + t0, t1 = steps or (0, H) + if not 0 <= t0 < t1 <= H or t0 % J or t1 % J: + raise ValueError("re-run steps must be J-aligned within the round") + if t0 and carry is None: + raise ValueError("re-run from a checkpoint needs its optimizer state") check_launch(lay, comm.world) traced = HookComm(comm, hook, cancel) comm = traced @@ -983,7 +1001,8 @@ def _train_island( adam_names = {n for n in theta_start if not uses_muon(cfg.inner, n)} if set(carry.m) != set(theta_start) or set(carry.v) != adam_names: raise ValueError("carried full state does not match the model") - if carry.step != a.global_step0: + base = a.global_step0 if cfg.inner.state_policy == "carry" or not t0 else 0 + if carry.step != base + t0: raise ValueError("carried optimizer step differs from assignment") if any( x.dtype != torch.float32 or x.shape != theta_start[n].shape @@ -998,7 +1017,7 @@ def _train_island( and lay.ep_size != 1 ): raise ValueError("ep_size incompatible with the model") - H, J, mb, ga = cfg.inner.H, cfg.inner.J, cfg.inner.micro_batch, cfg.inner.grad_accum + mb, ga = cfg.inner.micro_batch, cfg.inner.grad_accum if len(a.sample_ids) != H * mb * ga * lay.n_gpus: raise ValueError("assignment length must equal H * micro_batch * grad_accum * n_gpus") ia = IslandAssignment(a.run_id, a.w, a.sample_ids, a.global_step0, ranks=lay.n_gpus) @@ -1014,7 +1033,11 @@ def _train_island( carry.step, ) local_v0 = None if v0 is None else {n: geo.local(n, v0[n]) for n in owned if n in v0} - st = init_state(cfg.inner, {n: theta[n] for n in owned}, local_carry, local_v0) + if t0: + assert local_carry is not None + st = local_carry.clone() # mid-round state, whatever the round-start policy + else: + st = init_state(cfg.inner, {n: theta[n] for n in owned}, local_carry, local_v0) pieces = geo.pieces() adam = [p for p in pieces if not uses_muon(cfg.inner, p[0])] device = theta[names[0]].device @@ -1037,14 +1060,19 @@ def full() -> tuple[Params, OptState]: v = _assemble(geo, gather_pieces(comm, geo, st.v, adam, geo.owner, device), adam_names) return th, OptState(m, v, st.step) + # A re-run's starting gather is window l-1's closing event, never window l's. + traced.step, traced.hook = t0, None if t0 else hook th_full, st_full = full() - leaves = [_make_leaf(cfg, ia, 0, th_full, st_full, 0.0, 0.0)] - if checkpoint is not None: + traced.hook = hook + leaves = [] if t0 else [_make_leaf(cfg, ia, 0, th_full, st_full, 0.0, 0.0)] + if checkpoint is not None and not t0: checkpoint(0, th_full, st_full) window_start = th_full del th_full, st_full # memory: only window_start (theta) is needed between leaves per = mb * ga - for t in range(1, H + 1): + for t in range(t0 + 1, t1 + 1): + if hook_until is not None and t > hook_until: + hook = traced.hook = None traced.step = t if cancel is not None: cancel() @@ -1115,6 +1143,7 @@ def full() -> tuple[Params, OptState]: window_start = th_full del th_full, st_full del window_start + traced.hook = None # the closing gather is no window's event: every window has one shape th_full, st_full = full() if ef_in is None: ef_in = {n: torch.zeros_like(x) for n, x in th_full.items()} diff --git a/tests/auditor/test_island_lazy_window.py b/tests/auditor/test_island_lazy_window.py new file mode 100644 index 0000000..8aba90f --- /dev/null +++ b/tests/auditor/test_island_lazy_window.py @@ -0,0 +1,80 @@ +"""Lazy per-window traces: synthetic captures only, no training.""" + +from __future__ import annotations + +import pytest +import torch + +import hypertrain.trainer # noqa: F401 +from hypertrain.auditor.bisect import RefereeError +from hypertrain.auditor.island_bisect import IslandCapture, IslandParty, TraceEntry +from hypertrain.auditor.replay import pack_state +from hypertrain.protocol.keys import Keypair +from hypertrain.trainer.optim import OptState + +J, H, RANKS = 2, 6, 2 + + +def entries(rank: int, lo: int, salt: str = "") -> list[TraceEntry]: + return [ + TraceEntry( + rank=rank, + step=t, + microbatch=-1, + layer=-1, + op=op, + shape=[t], # data-dependent shapes (MoE rows) must not break the skeleton check + dtype="torch.float32", + sha256=(f"{rank}{t}{op}{salt}".encode().hex() * 64)[:64], + ) + for t in range(lo + 1, lo + J + 1) + for op in ("all_gather.input", "optimizer.theta.w") + ] + + +def captures() -> list[IslandCapture]: + out = [] + for rank in range(RANKS): + c = IslandCapture() + c.entries = entries(rank, 0) # commit path: window 1 probe only + if rank == 0: # checkpoints exist under rank-0 only + for t in range(0, H + 1, J): + + def w(t: int = t) -> dict[str, torch.Tensor]: + return {"w": torch.full((2,), float(t))} + + c.states[t] = pack_state(w(), OptState(w(), w(), t)) + out.append(c) + return out + + +def test_lazy_window_reruns_once_and_matches_continuous_trace() -> None: + calls: list[int] = [] + + def retrace(leaf: int) -> list[list[TraceEntry]]: + calls.append(leaf) + return [entries(r, (leaf - 1) * J) for r in range(RANKS)] + + party = IslandParty(Keypair(b"\x71" * 32).ss58, captures(), J=J, retrace=retrace) + continuous = captures() + for rank, c in enumerate(continuous): + c.entries = [e for lo in range(0, H, J) for e in entries(rank, lo)] + full = IslandParty(Keypair(b"\x72" * 32).ss58, continuous, J=J) + # Sizing a dispute never re-runs; every window has window 1's shape. + assert [party.span("layer", (w,)) for w in (1, 2, 3)] == [ + full.span("layer", (w,)) for w in (1, 2, 3) + ] + assert calls == [] + at = list(range(party.span("layer", (3,)) + 1)) + assert party.hashes("layer", (3,), at) == full.hashes("layer", (3,), at) + party.hashes("op", (3, 0), [0, 1]) + assert calls == [3] + + +def test_lazy_window_rejects_rerun_with_a_different_event_shape() -> None: + def retrace(leaf: int) -> list[list[TraceEntry]]: + return [entries(r, (leaf - 1) * J)[:-1] for r in range(RANKS)] + + party = IslandParty(Keypair(b"\x73" * 32).ss58, captures(), J=J, retrace=retrace) + with pytest.raises(RefereeError, match="probe window shape"): + party.window(2) diff --git a/tests/data/test_parts.py b/tests/data/test_parts.py index 0f7fec0..150f38e 100644 --- a/tests/data/test_parts.py +++ b/tests/data/test_parts.py @@ -1,6 +1,7 @@ import hashlib import json import os +import threading from pathlib import Path import pytest @@ -257,3 +258,48 @@ def get(sha: str) -> bytes: fetch_to(ref, get, out / "dest", workers=2) # Window is 4 * workers = 8 parts: the 9th is never requested. assert os.listdir(out) == [] and index.parts[-1] not in asked + + +def test_parallel_publish_puts_parts_concurrently_and_index_last(tmp_path: Path) -> None: + # Given: a 9-part file; part 0's PUT only completes once part 1's PUT has started, so a + # serial publisher would fail the wait instead of passing. + data = _blob(8 * PART_BYTES + 77) + src = tmp_path / "f" + src.write_bytes(data) + serial = Recorder(tmp_path / "serial") + want = put_file(src, serial.put) + rec, second_started, lock = Recorder(tmp_path / "s"), threading.Event(), threading.Lock() + + def put(chunk: bytes) -> str: + if chunk == data[:PART_BYTES]: + assert second_started.wait(timeout=10), "parts were not put concurrently" + if chunk == data[PART_BYTES : 2 * PART_BYTES]: + second_started.set() + with lock: + return rec.put(chunk) + + # When: published with 4 workers. + ref = put_file(src, put, workers=4) + # Then: same ref as serial, index last, and it lists the parts in file order. + assert ref == want and parse_index(rec.puts[-1]) == parse_index(serial.puts[-1]) + assert all(parse_index(b) is None for b in rec.puts[:-1]) + assert get_large(ref, rec.store.get) == data + + +def test_parallel_publish_failing_part_never_puts_index(tmp_path: Path) -> None: + data = _blob(8 * PART_BYTES + 1) + src = tmp_path / "f" + src.write_bytes(data) + rec, lock = Recorder(tmp_path / "s"), threading.Lock() + + def put(chunk: bytes) -> str: + if chunk == data[3 * PART_BYTES : 4 * PART_BYTES]: + raise StoreError("boom") + with lock: + return rec.put(chunk) + + with pytest.raises(StoreError, match="boom"): + put_file(src, put, workers=2) + # Window is 2 * workers = 4 parts in flight: no index, and the tail is never queued. + assert all(parse_index(b) is None for b in rec.puts) + assert data[-1:] not in rec.puts diff --git a/tests/datasets/test_datasets.py b/tests/datasets/test_datasets.py index 0df5b78..e44b677 100644 --- a/tests/datasets/test_datasets.py +++ b/tests/datasets/test_datasets.py @@ -247,6 +247,12 @@ def do_GET(self) -> None: self.send_error(500) return data = (Path(self.directory) / self.path.lstrip("/")).read_bytes() # type: ignore[arg-type] + if self.headers["Range"] is None: # plain GET (dataset manifest.json) + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return a, b = self.headers["Range"].removeprefix("bytes=").split("-") body = data[int(a) : int(b) + 1] if self.server.corrupt: # type: ignore[attr-defined] @@ -380,3 +386,22 @@ def test_shard_sampler_u16_branch(tmp_path: Path) -> None: assert get(7).tolist() == s.row(7).tolist() and get(7).dtype == np.uint32 with pytest.raises(MinerError, match="unit root"): shard_sampler(tmp_path / "src", _manifest_for(m, unit_sha256_root="1" * 64)) + + +def test_shard_sampler_u16_remote_units(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + m, pub = _published(tmp_path) + (pub / "manifest.json").write_text(m.to_json()) + srv, url = _serve(pub) + try: + monkeypatch.setenv("HYPERTRAIN_DATA_BASES", url) + get = shard_sampler(tmp_path / "remote", _manifest_for(m)) # no local shards + s = U16ShardSamples(tmp_path / "src", m.n_shards, m.samples_per_shard, m.seq_len) + assert get(13).tolist() == s.row(13).tolist() and get(13).dtype == np.uint32 + assert (tmp_path / "remote/units" / m.unit_sha256s[13 // m.unit]).is_file() + forged = m.to_json().replace(m.unit_sha256s[0], "0" * 64) + (tmp_path / "forged").mkdir() + (tmp_path / "forged/manifest.json").write_text(forged) + with pytest.raises(MinerError, match="signed roots"): + shard_sampler(tmp_path / "forged", _manifest_for(m)) + finally: + srv.shutdown() diff --git a/tests/gpu_ops/test_network_gpu_qualification.py b/tests/gpu_ops/test_network_gpu_qualification.py index 9c29bb5..4fac756 100644 --- a/tests/gpu_ops/test_network_gpu_qualification.py +++ b/tests/gpu_ops/test_network_gpu_qualification.py @@ -265,7 +265,7 @@ def test_partial_round_coverage_rejects_even_identical_results(tmp_path: Path) - paths = results(tmp_path) for path in paths: data = json.loads(path.read_bytes()) - del data["rounds"][0]["rank-1/checkpoints/30.safetensors"] + del data["rounds"][0]["rank-0/checkpoints/30.safetensors"] path.write_text(json.dumps(data)) # When / Then with pytest.raises(Reject, match="coverage_missing"): @@ -277,10 +277,11 @@ def test_eight_gpu_h4_j2_results_compare_with_full_rank_coverage(tmp_path: Path) paths = results(tmp_path, ranks=8, H=4, J=2) # When result = driver.compare(paths, manifest(8, 4, 2)) - # Then: 8 ranks x (5 finals + checkpoints 0,2,4), workload gate unchanged. + # Then: 8 ranks x 5 finals + rank-0 checkpoints 0,2,4 (only rank 0 writes them). rounds = json.loads(paths[0].read_bytes())["rounds"] assert result == {"two_host_bitwise_match": True, "workload_allowed": False} - assert [len(r) for r in rounds] == [64, 64] and "rank-7/checkpoints/4.safetensors" in rounds[1] + assert [len(r) for r in rounds] == [43, 43] and "rank-0/checkpoints/4.safetensors" in rounds[1] + assert not any(k.startswith("rank-7/checkpoints/") for k in rounds[1]) @pytest.mark.parametrize("attack", ["missing_rank", "same_host", "tamper", "wrong_image"]) @@ -303,7 +304,7 @@ def test_eight_gpu_results_reject_rank_host_custody_and_shape_attacks( for path, data in zip(paths, datas, strict=True): path.write_text(json.dumps(data)) if attack == "tamper": - (paths[1].parent / "round-1/published/rank-7/checkpoints/2.safetensors").write_bytes(b"x") + (paths[1].parent / "round-1/published/rank-0/checkpoints/2.safetensors").write_bytes(b"x") match = {"missing_rank": "coverage_missing", "tamper": "custody_changed"} # When / Then with pytest.raises(Reject, match=match.get(attack, "binding_or_bitwise")): @@ -370,7 +371,7 @@ def test_eight_gpu_seed_manifest_drives_rank_and_checkpoint_coverage( captured = driver.capture(published, job) # Then assert len(job.sample_ids) == 32 and set(captured) == set(driver.round_files(8, 4, 2)) - (published / "rank-7/checkpoints/4.safetensors").unlink() + (published / "rank-0/checkpoints/4.safetensors").unlink() with pytest.raises(FileNotFoundError): driver.capture(published, job) diff --git a/tests/gpu_ops/test_network_lifecycle.py b/tests/gpu_ops/test_network_lifecycle.py index 3bf0fff..9ae1f88 100644 --- a/tests/gpu_ops/test_network_lifecycle.py +++ b/tests/gpu_ops/test_network_lifecycle.py @@ -615,9 +615,10 @@ def test_eight_gpu_od_large_budget_is_geometry_derived() -> None: bounds = mod.artifact_budget(od_large_profile(3)) assert bounds["param_count"] == 353861632 and bounds["tensor_count"] == 339 # Task 19 measured: state 4246444200, ef 1415481880, delta 359411534, - # eight-rank execution output 150085361712 bytes; bounds must cover all of them. + # eight-rank execution output 150085361712 bytes with checkpoints on every rank. Checkpoints + # are now rank-0 only: 3 x state + 8 x (state + ef + delta) = 60910033512 bytes. assert bounds["state_bytes"] >= 4246444200 - 65536 - assert bounds["execution_bytes"] >= 150085361712 + assert bounds["execution_bytes"] >= 3 * 4246444200 + 8 * (4246444200 + 1415481880 + 359411534) assert bounds["retained_bytes_per_host"] == 3 * bounds["execution_bytes"] assert mod.execution_quota(od_large_profile(3))["remaining"] == 2 for change in ( @@ -681,9 +682,16 @@ def test_prune_rescued_policy_budget_bounds_and_requires_local_custody() -> None assert bounds["disk_artifacts_peak_bytes"] < 400_000_000_000 # 7 per host = 2 qualification + 5 workload executions per host (10 total) within 4.7TB. assert bounds["local_custody_bytes_required"] < 4_700_000_000_000 + # Dedup rescue: each workload keeps one rank's tensors + all traces (~33GB, not ~170GB), + # so 8 per host now fits; the bound still grows with the schedule and still binds. larger = od_large_profile(8) larger["artifacts"] |= {"remote_retention": "prune_rescued"} - assert mod.artifact_budget(larger)["local_custody_bytes_required"] > 4_700_000_000_000 + assert mod.artifact_budget(larger)["local_custody_bytes_required"] < 4_700_000_000_000 + # Rank-0-only checkpoints already removed most rank redundancy from execution_bytes. + assert bounds["execution_transfer_bytes"] < bounds["execution_bytes"] // 2 + largest = od_large_profile(40) + largest["artifacts"] |= {"remote_retention": "prune_rescued"} + assert mod.artifact_budget(largest)["local_custody_bytes_required"] > 4_700_000_000_000 for change in ( {"remote_retention": "delete"}, {"remote_retention": "prune_rescued", "retain_all_checkpoints": False}, @@ -707,8 +715,10 @@ def test_prune_rescued_policy_budget_bounds_and_requires_local_custody() -> None ready = mod.readiness_budget(pruned | {"artifacts": artifacts}, evidence) assert ready["remote_retention"] == "prune_rescued" # The image and staging the host still needs are part of the 400GB check, not waived. - artifacts |= {"image_and_dependency_bytes": 60_000_000_000} - evidence |= {"image_and_dependency_bytes": 60_000_000_000} + # Rank-0-only checkpoints shrank the peak, so it takes a larger image to overflow 400GB. + image = 400_000_000_000 - bounds["disk_artifacts_peak_bytes"] + artifacts |= {"image_and_dependency_bytes": image} + evidence |= {"image_and_dependency_bytes": image} with pytest.raises(Reject, match="artifact_disk_budget_exceeded"): mod.readiness_budget(pruned | {"artifacts": artifacts}, evidence) @@ -956,8 +966,9 @@ def test_hash_only_export_carries_hashes_and_small_json_only(tmp_path: Path) -> with tarfile.open(archive) as t: carried = {m.name for m in t.getmembers()} # Every remote file is hashed; only result/config/job/summary bytes travel. - # 2 rounds x (2 ranks x 12 round files + 2 summaries + job.json) + config + result. - assert len(files) == 2 * (2 * 12 + 2 + 1) + 2 + # 2 rounds x (rank-0 12 + rank-1 5 round files (checkpoints rank-0 only) + 2 summaries + # + job.json) + config + result. + assert len(files) == 2 * (12 + 5 + 2 + 1) + 2 assert carried == {"manifest.json"} | { "out/" + n for n in files if Path(n).name in mod.HASH_RESCUE_NAMES } @@ -1060,12 +1071,70 @@ def argv(ssh, command): carried = {m.name for m in t.getmembers()} state = "out/qualification/round-0/published/rank-0/state.safetensors" assert record["hashes_only"] is (not workload) - assert commands[0].endswith("profile.json" if workload else "profile.json hashes") + # After a workload the rescue is deduplicated (each distinct content once), never full. + assert commands[0].endswith("profile.json dedup" if workload else "profile.json hashes") + assert record["dedup_known"] == ([] if workload else None) assert (state in carried) is workload # hash-only never ships the tensor bytes assert state.removeprefix("out/") in record["files"] # but always its remote hash assert runtime.rescue("h0") == record # resume re-verifies in the recorded mode +def dedup_op_tree(tmp_path: Path) -> tuple[Path, str]: + """A 2-rank operation output: identical rank tensors/checkpoints, distinct traces.""" + src = tmp_path / "remote-out" + pub = src / "op-a/published" + for rank in range(2): + d = pub / f"rank-{rank}" + (d / "checkpoints").mkdir(parents=True) + for f in ("state.safetensors", "ef.safetensors", "delta.bin", "leaves.json"): + (d / f).write_bytes(f.encode() * 64) + for t in (0, 5): + (d / f"checkpoints/{t}.safetensors").write_bytes(b"c%d" % t * 64) + (d / "trace.json").write_bytes(b"trace-%d" % rank) + (d / "summary.json").write_bytes(b'{"rank": %d}' % rank) + (pub / "start_state").write_bytes(b"input" * 64) + (src / "op-a/start_state").write_bytes(b"input" * 64) + return src, fsha(pub / "start_state") + + +def test_dedup_export_carries_rank0_once_plus_every_rank_hash(tmp_path: Path) -> None: + import tarfile + + mod = network_module() + src, known = dedup_op_tree(tmp_path) + profile = json.loads((TREE / "experiments/gpu_network_v2/profile.json").read_bytes()) + archive = tmp_path / "dedup.tar" + mod.export_artifacts(src, archive, profile, False, [known]) + files = mod.verify_export(archive, 1 << 30, False, [known]) + with tarfile.open(archive) as t: + carried = {m.name.removeprefix("out/") for m in t.getmembers()} - {"manifest.json"} + pub = "op-a/published/" + # Every remote file keeps its host-computed hash; bytes travel once per distinct content. + assert len(files) == 2 * 8 + 2 # per rank: 4 tensors + 2 checkpoints + trace + summary + assert not any( + "rank-1/" in n and not n.endswith(("trace.json", "summary.json")) for n in carried + ) + assert {pub + "rank-0/state.safetensors", pub + "rank-0/checkpoints/5.safetensors"} <= carried + assert {pub + "rank-1/trace.json", pub + "rank-1/summary.json"} <= carried + assert not any(n.endswith("start_state") for n in carried) # held locally: hash only + assert not mod.rank_bytes_differ(files) + # A dedup archive is not a full rescue, and the dedup verifier refuses an extra member. + with pytest.raises(Reject, match="coverage_mismatch"): + mod.verify_export(archive, 1 << 30) + with pytest.raises(Reject, match="coverage_mismatch"): + mod.verify_export(archive, 1 << 30, False, []) + # A rank whose state/checkpoint hash differs from rank-0's is detected from hashes alone. + for rel in ("state.safetensors", "checkpoints/5.safetensors"): + bad = {k: dict(v) for k, v in files.items()} + bad[pub + "rank-1/" + rel]["sha256"] = "ab" * 32 + assert mod.rank_bytes_differ(bad) + bad = {k: v for k, v in files.items() if k != pub + "rank-0/ef.safetensors"} + assert mod.rank_bytes_differ(bad) + traces = {k: dict(v) for k, v in files.items()} + traces[pub + "rank-1/trace.json"]["sha256"] = "cd" * 32 + assert not mod.rank_bytes_differ(traces) # traces legitimately differ per rank + + def test_verified_dedup_hardlinks_identical_bytes_only(tmp_path: Path) -> None: mod = network_module() seen: dict[str, Path] = {} diff --git a/tests/gpu_ops/test_network_role_dispatch.py b/tests/gpu_ops/test_network_role_dispatch.py index 10788d4..3eb1cbe 100644 --- a/tests/gpu_ops/test_network_role_dispatch.py +++ b/tests/gpu_ops/test_network_role_dispatch.py @@ -333,6 +333,7 @@ def rescue_run(ssh, command, tag, logs): + " " + shlex.quote(runtime.lifecycle.remote_root("h0") + "/rescue.tar") + " experiments/gpu_network_v2/profile.json" + + command.split("profile.json", 1)[1] # the runtime's own mode (dedup ...) ) return original_run(ssh, command, tag, logs) diff --git a/tests/miner/test_client_network_v2.py b/tests/miner/test_client_network_v2.py index f08bfea..a07a4b8 100644 --- a/tests/miner/test_client_network_v2.py +++ b/tests/miner/test_client_network_v2.py @@ -406,3 +406,129 @@ def flaky(request: httpx.Request) -> httpx.Response: with pytest.raises(ChallengeHTTPError, match="502"): miner.api.call("POST", "/v2/runs/x/commit", json={}) assert hits == ["POST"] + + +def test_fetch_inputs_reuses_own_carry_and_downloads_only_theta(tmp_path: Path): + import hashlib + from types import SimpleNamespace + + import torch + + from hypertrain.auditor.replay import pack_state + from hypertrain.data.store import put_large + from hypertrain.trainer.optim import OptState + + # Given: the miner's own previous publication (theta_old + m/v/step, ef) and a new global + # theta published by the master as a theta-only object. + miner, store, _ = _parts_miner(tmp_path) + n = 300_000 # > 1 MiB per tensor: indexed objects + carry = OptState({"w": torch.full((n,), 0.5)}, {"w": torch.full((n,), 0.25)}, 7) + previous = tmp_path / "prev" + previous.mkdir() + (previous / "state.safetensors").write_bytes(pack_state({"w": torch.zeros(n)}, carry)) + ef = pack_state({"w": torch.ones(n)}) + (previous / "ef.safetensors").write_bytes(ef) + theta = {"w": torch.arange(n, dtype=torch.float32)} + start = pack_state(theta, carry) + theta_ref = put_large(pack_state(theta), store.put) + full = {"start_state": put_large(start, store.put), "ef_in": put_large(ef, store.put)} + job = SimpleNamespace( + start_state_sha256=hashlib.sha256(start).hexdigest(), + ef_in_sha256=hashlib.sha256(ef).hexdigest(), + v0_sha256="00" * 32, + object_paths={"start_state": "start_state", "ef_in": "ef_in"}, + ) + gets: list[str] = [] + fetch = miner._get_object + miner._get_object = lambda digest: gets.append(digest) or fetch(digest) + directory = tmp_path / "job" + directory.mkdir() + # When: inputs are fetched with the local carry available. + miner._fetch_inputs(job, full, directory, theta_ref, previous) + # Then: byte-identical inputs; only the theta object was downloaded. + assert (directory / "start_state").read_bytes() == start + assert (directory / "ef_in").read_bytes() == ef + assert gets[0] == theta_ref and not set(gets) & set(full.values()) + assert sorted(p.name for p in directory.iterdir()) == ["ef_in", "start_state"] + + +def test_fetch_inputs_falls_back_to_full_download_when_local_carry_differs(tmp_path: Path): + import hashlib + from types import SimpleNamespace + + import torch + + from hypertrain.auditor.replay import pack_state + from hypertrain.data.store import put_large + from hypertrain.trainer.optim import OptState + + # Given: a stale local publication whose m/v and ef differ from the master's pins. + miner, store, _ = _parts_miner(tmp_path) + n = 300_000 + previous = tmp_path / "prev" + previous.mkdir() + stale = OptState({"w": torch.zeros(n)}, {"w": torch.zeros(n)}, 1) + (previous / "state.safetensors").write_bytes(pack_state({"w": torch.zeros(n)}, stale)) + (previous / "ef.safetensors").write_bytes(pack_state({"w": torch.zeros(n)})) + theta = {"w": torch.arange(n, dtype=torch.float32)} + start = pack_state(theta, OptState({"w": torch.ones(n)}, {"w": torch.ones(n)}, 2)) + ef = pack_state({"w": torch.ones(n)}) + full = {"start_state": put_large(start, store.put), "ef_in": put_large(ef, store.put)} + job = SimpleNamespace( + start_state_sha256=hashlib.sha256(start).hexdigest(), + ef_in_sha256=hashlib.sha256(ef).hexdigest(), + v0_sha256="00" * 32, + object_paths={"start_state": "start_state", "ef_in": "ef_in"}, + ) + directory = tmp_path / "job" + directory.mkdir() + # When / Then: the pinned full objects are downloaded instead; no temp file remains. + miner._fetch_inputs(job, full, directory, put_large(pack_state(theta), store.put), previous) + assert (directory / "start_state").read_bytes() == start + assert (directory / "ef_in").read_bytes() == ef + assert sorted(p.name for p in directory.iterdir()) == ["ef_in", "start_state"] + + +@pytest.mark.parametrize("bad", ["missing", "corrupt", "wrong_shape"]) +def test_fetch_inputs_falls_back_to_full_download_on_bad_theta_ref(tmp_path: Path, bad: str): + import hashlib + from types import SimpleNamespace + + import torch + + from hypertrain.auditor.replay import pack_state + from hypertrain.data.store import put_large + from hypertrain.trainer.optim import OptState + + # Given: a matching local carry, but the theta ref is unfetchable, corrupt or mis-shaped. + miner, store, _ = _parts_miner(tmp_path) + n = 300_000 + carry = OptState({"w": torch.full((n,), 0.5)}, {"w": torch.full((n,), 0.25)}, 7) + previous = tmp_path / "prev" + previous.mkdir() + (previous / "state.safetensors").write_bytes(pack_state({"w": torch.zeros(n)}, carry)) + theta = {"w": torch.arange(n, dtype=torch.float32)} + start = pack_state(theta, carry) + theta_ref = { + "missing": "ab" * 32, + "corrupt": store.put(b"not a safetensors file"), + "wrong_shape": put_large(pack_state({"w": torch.ones(n + 1)}), store.put), + }[bad] + full = {"start_state": put_large(start, store.put)} + job = SimpleNamespace( + start_state_sha256=hashlib.sha256(start).hexdigest(), + ef_in_sha256="00" * 32, + v0_sha256="00" * 32, + object_paths={"start_state": "start_state"}, + ) + gets: list[str] = [] + fetch = miner._get_object + miner._get_object = lambda digest: gets.append(digest) or fetch(digest) + directory = tmp_path / "job" + directory.mkdir() + # When: inputs are fetched. + miner._fetch_inputs(job, full, directory, theta_ref, previous) + # Then: no MinerError; the pinned full object was downloaded; no temp file remains. + assert (directory / "start_state").read_bytes() == start + assert full["start_state"] in gets + assert sorted(p.name for p in directory.iterdir()) == ["start_state"] diff --git a/tests/protocol/test_network_v2.py b/tests/protocol/test_network_v2.py index 40eaeb4..1ef100e 100644 --- a/tests/protocol/test_network_v2.py +++ b/tests/protocol/test_network_v2.py @@ -1078,3 +1078,34 @@ def test_v1_when_opaque_published_baseline_is_replayed(tmp_path: Path) -> None: ) assert tape_bytes(tape) == bytes.fromhex(tape_fixture["signed_hex"]) assert sha256_hex(result.to_bytes()) == tape["body"]["out_state"] + + +def test_audit_lease_when_h_scales_with_step_bound() -> None: + from hypertrain.protocol.messages import f32hex + from hypertrain.protocol.messages_v2 import AuditPolicyV2, audit_lease_rounds + + # Given: the pinned small-H lease and the H=512 / 4 s production bound. + assert audit_lease_rounds(30, 4.0) == 100 + assert audit_lease_rounds(512, 4.0) == 1466 # ceil(2*512*4/3) + 100 + body = example_manifest().body() + body["model"]["param_count"] = 4_329_216 + body["inner"].update(H=512, J=64) + training = wrapper(body).training + policy = dict( + max_attempts=2, + max_concurrent_per_auditor=1, + max_running_jobs=2, + max_queued_jobs=32, + max_anchor_age_rounds=1, + max_steps_per_attempt=1024, + ) + # When / Then: only the derived lease validates; absolute cap is twice the lease. + ok = AuditPolicyV2(lease_rounds=1466, **policy) + ok.validate_training(training) + assert ok.absolute_rounds() == 2932 + with pytest.raises(ValueError, match="derived lease"): + AuditPolicyV2(lease_rounds=100, **policy).validate_training(training) + with pytest.raises(ValueError, match="step_seconds_bound"): + AuditPolicyV2( + lease_rounds=1466, step_seconds_bound=f32hex(float("nan")), **policy + ).validate_training(training) From 13293e0fa458e2a0e0425d173bccbbb1c0378443 Mon Sep 17 00:00:00 2001 From: hypertrain-bot Date: Thu, 15 Oct 2026 19:25:11 +0000 Subject: [PATCH 4/5] fix(types): mypy-clean dataset loader and lazy-window backend narrowing --- src/hypertrain/datasets/build.py | 9 +++++---- src/hypertrain/miner/island_launch.py | 12 +++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/hypertrain/datasets/build.py b/src/hypertrain/datasets/build.py index 505098b..c8f5766 100644 --- a/src/hypertrain/datasets/build.py +++ b/src/hypertrain/datasets/build.py @@ -75,11 +75,12 @@ def load(src: Source) -> Iterable[Row]: if p.suffix == ".zst": import zstandard - raw = io.TextIOWrapper(zstandard.ZstdDecompressor().stream_reader(p.open("rb"))) + zr = zstandard.ZstdDecompressor().stream_reader(p.open("rb")) + with io.TextIOWrapper(zr) as raw: + yield from (json.loads(line) for line in raw if line.strip()) else: - raw = io.TextIOWrapper(gzip.open(p, "rb")) - with raw: - yield from (json.loads(line) for line in raw if line.strip()) + with gzip.open(p, "rt") as gz: + yield from (json.loads(line) for line in gz if line.strip()) return load diff --git a/src/hypertrain/miner/island_launch.py b/src/hypertrain/miner/island_launch.py index c290f00..054a429 100644 --- a/src/hypertrain/miner/island_launch.py +++ b/src/hypertrain/miner/island_launch.py @@ -647,8 +647,14 @@ def launch_window(job: IslandJobV1, published: Path, leaf: int, *, timeout: floa n_gpus = job.manifest.training.reference_spec.layout.n_gpus outputs = [published / f"rank-{r}/windows/{leaf}.json" for r in range(n_gpus)] summary = load_json(confined(published, "rank-0/summary.json").read_bytes()) - if not isinstance(summary, dict) or summary.get("backend") not in ("cpu", "cuda"): - raise IslandFailure("publication backend missing") + backend: Literal["cpu", "cuda"] + match summary.get("backend") if isinstance(summary, dict) else None: + case "cpu": + backend = "cpu" + case "cuda": + backend = "cuda" + case _: + raise IslandFailure("publication backend missing") ref = job.manifest.training.reference_spec env = dict( os.environ, @@ -662,7 +668,7 @@ def launch_window(job: IslandJobV1, published: Path, leaf: int, *, timeout: floa if all(p.is_file() for p in outputs): return proc = subprocess.Popen( # noqa: S603 - fixed interpreter/package, typed layout - launch_argv(job, published, summary["backend"]), + launch_argv(job, published, backend), env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, From b0d5e5813459842aa31274b72805d5c805de36fd Mon Sep 17 00:00:00 2001 From: hypertrain-bot Date: Thu, 15 Oct 2026 19:32:08 +0000 Subject: [PATCH 5/5] test: rescue carries rank-0 checkpoints only --- tests/miner/test_island_launch_v2.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/miner/test_island_launch_v2.py b/tests/miner/test_island_launch_v2.py index b39486a..168e042 100644 --- a/tests/miner/test_island_launch_v2.py +++ b/tests/miner/test_island_launch_v2.py @@ -557,7 +557,8 @@ def test_exact_staged_contract_over_real_ssh(tmp_path: Path) -> None: assert len(journal.all("network_job_started")) == 1 rescued = runtime.rescue("h0") assert any("rank-0/checkpoints/" in name for name in rescued["files"]) - assert any("rank-1/checkpoints/" in name for name in rescued["files"]) + # checkpoints are written by rank 0 only (ranks are bitwise identical) + assert not any("rank-1/checkpoints/" in name for name in rescued["files"]) assert lifecycle.rescue("h0") == "verified" assert not journal.all("provider_call") finally: