diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6b1884986..49d3f28dd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -56,6 +56,9 @@ updates: - "/src/agent_env/agents/agents/pydantic-ai" - "/src/agent_env/examples/mail-mcp/envs/mail" - "/src/agent_env/computers/artifacts/ubuntu-24.04" + - "/src/agent_env/computers/disk/linux/boot" + - "/src/agent_env/computers/disk/linux/tools" + - "/src/agent_env/computers/disk/linux/launcher" schedule: interval: "weekly" open-pull-requests-limit: 5 diff --git a/.github/scripts/check_clean_install.py b/.github/scripts/check_clean_install.py index 2be81aee9..0b0c6c54f 100644 --- a/.github/scripts/check_clean_install.py +++ b/.github/scripts/check_clean_install.py @@ -15,6 +15,7 @@ import argparse import compileall +import dataclasses import importlib.metadata import json import os @@ -85,10 +86,8 @@ def problems(name: str, runs: int, before: dict[str, list[str]]) -> list[str]: found.append(f"agent-env found the config file {config}; the gate runs with none") for plugin in plugins(): found.append(f"{plugin} is installed; the gate tests agent-env without plugins") - installed_src = Path(agent_env.__file__).parent.parent for image in PUBLISHED_IMAGES: - checkout = PublishedImage(image.name, CHECKOUT_SRC / image.dockerfile.relative_to(installed_src)) - if (tag := image.tag()) != (expected := checkout.tag()): + if (tag := image.tag()) != (expected := _checkout(image).tag()): found.append(f"{image.name}: the installed sources tag it {tag}, the checkout's {expected}; an install " "wouldn't find the image published for its sources") bundle = find_bundle(name) @@ -121,6 +120,13 @@ def problems(name: str, runs: int, before: dict[str, list[str]]) -> list[str]: return found +def _checkout(image: PublishedImage) -> PublishedImage: + """``image`` as the checkout's sources make it, with the images it's built from.""" + installed_src = Path(agent_env.__file__).parent.parent + return dataclasses.replace(image, dockerfile=CHECKOUT_SRC / image.dockerfile.relative_to(installed_src), + args=tuple((arg, _checkout(built)) for arg, built in image.args)) + + def plugins() -> list[str]: """Entry points in an ``agent_env.*`` group from any distribution other than agent-env's own.""" return sorted(f"{dist.name}'s {ep.group} entry point {ep.name}" diff --git a/.github/scripts/infra_build.py b/.github/scripts/infra_build.py index c4ba84fa8..15d6e69a1 100644 --- a/.github/scripts/infra_build.py +++ b/.github/scripts/infra_build.py @@ -4,26 +4,39 @@ images agent-env ships before it's built, pushed or recorded. ``check ENTRY`` fails unless ENTRY, a JSON object as ``infra_images.py plan`` lists an image, names an image agent-env -ships, with its Dockerfile, context, repository and a source tag. ``exists REF`` exits 0 when the registry has REF, 1 -when it says it doesn't, and 2 when it can't tell. ``smoke NAME IMAGE`` checks that IMAGE, a local build of NAME, runs. -``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the registry serves -under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI doesn't serve -yet the agentenv-framework-protocol an agent image's build installs. +ships, with its Dockerfile, context, repository and a source tag, and for a disk form, the build args naming its base +computer's image and the disk tools by their repositories and source tags, and its size. ``exists REF`` exits 0 when +the registry has REF, 1 when it says it doesn't, and 2 when it can't tell. ``wanted ENTRY`` checks ENTRY, signs in to +GHCR and writes ``build=true`` to the step's output when the registry lacks its tag, ``build=false`` when it has it. +``build ENTRY ARCH`` checks ENTRY and builds it for linux/ARCH as its tag with ``-ARCH`` after it; a disk form from its +base computer's image of ARCH, which it pulls, with that image's config. ``config IMAGE`` prints the config object of +IMAGE's OCI config, base64'd, as a disk form's build takes its base's. ``smoke NAME IMAGE`` checks that IMAGE, a local +build of NAME, runs: a disk form boots in the QEMU launcher image ``--launcher`` names until every program its base's +supervisord.conf declares runs. ``publish MISSING`` signs in to GHCR and tags the two platform builds of each entry of +MISSING, a JSON list, as one multi-platform image under its tag, once the entry is checked, unless the registry has +that tag. ``verify-record PATH`` fails unless PATH records each image agent-env ships once, with the digest the +registry serves under its tag now. ``retry COMMAND...`` runs COMMAND, again while a registry rate-limits it or PyPI +doesn't serve yet the agentenv-framework-protocol an agent image's build installs. """ from __future__ import annotations import argparse +import base64 import json +import os +import platform import posixpath import re import subprocess import sys import time +import tomllib from collections.abc import Callable from pathlib import Path REGISTRY = "ghcr.io/scaleapi" +REPO = Path(__file__).resolve().parents[2] # Each image agent-env ships, by name, and the Dockerfile it's built from, with its folder as the context. IMAGES = { "agentenv-gateway": "src/agent_env/env/gateway/Dockerfile", @@ -43,12 +56,23 @@ "agentenv-pydantic-ai": "src/agent_env/agents/agents/pydantic-ai/Dockerfile", "agentenv-mail-mcp": "src/agent_env/examples/mail-mcp/envs/mail/Dockerfile", "agentenv-ubuntu-24.04": "src/agent_env/computers/artifacts/ubuntu-24.04/Dockerfile", + "agentenv-disk-tools": "src/agent_env/computers/disk/linux/tools/Dockerfile", + "agentenv-qemu-launcher": "src/agent_env/computers/disk/linux/launcher/Dockerfile", + "agentenv-ubuntu-24.04-disk": "src/agent_env/computers/disk/linux/boot/Dockerfile", } +# Each disk form's build args, by the images agent-env ships that they name; its SIZE_GB is its base's [forms.disk] +# min_disk_gb, read from the artifact.toml beside the base's Dockerfile. +DISK_ARGS = {"agentenv-ubuntu-24.04-disk": {"ROOTFS": "agentenv-ubuntu-24.04", "TOOLS": "agentenv-disk-tools"}} _TAG = re.compile(r"^src-[0-9a-f]{64}$") _DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$") _HEALTHY_WITHIN_SECONDS = 120 _HEALTH_POLL_SECONDS = 2 _LOG_TAIL_LINES = 30 # of a container that never turned healthy, in the error +# How long a disk form may take to boot to its desktop: seconds with KVM, minutes under TCG. +_BOOTED_WITHIN_SECONDS = 900 +# What supervisord prints on a disk form's console as each program starts. +_RUNNING = re.compile(r"success: (\S+) entered RUNNING") +_PROGRAM = re.compile(r"^\[program:([^\]]+)\]", re.MULTILINE) _PRINTED_CHARS = 200 # of what docker printed in place of a manifest, in the error # What a registry answers when it rate-limits a request: ECR Public's 429, and the toomanyrequests error code GHCR and # Docker Hub send. @@ -94,6 +118,11 @@ " s=$(supervisorctl -c /etc/supervisor/supervisord.conf status);" " echo \"$s\" | grep -q ' RUNNING ' && ! echo \"$s\" | grep -qv ' RUNNING ' && exit 0;" " done; supervisorctl -c /etc/supervisor/supervisord.conf status; exit 1"], + "agentenv-disk-tools": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", + "command -v disk image-env mkfs.ext4 mkfs.vfat mcopy sfdisk qemu-img"], + "agentenv-qemu-launcher": ["docker", "run", "--rm", "--entrypoint", "sh", "{image}", "-c", + "qemu-system-x86_64 --version && qemu-system-aarch64 --version" + " && test -f /usr/share/OVMF/OVMF_CODE_4M.fd && test -f /usr/share/AAVMF/AAVMF_CODE.fd"], } @@ -102,9 +131,16 @@ def main() -> int: commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("check").add_argument("entry") commands.add_parser("exists").add_argument("ref") + commands.add_parser("wanted").add_argument("entry") + build = commands.add_parser("build") + build.add_argument("entry") + build.add_argument("arch", choices=("amd64", "arm64")) + commands.add_parser("config").add_argument("image") smoke = commands.add_parser("smoke") smoke.add_argument("name", choices=sorted(IMAGES)) smoke.add_argument("image") + smoke.add_argument("--launcher") + commands.add_parser("publish").add_argument("missing") commands.add_parser("verify-record").add_argument("path", type=Path) commands.add_parser("retry").add_argument("argv", nargs=argparse.REMAINDER, metavar="COMMAND") args = parser.parse_args() @@ -112,14 +148,98 @@ def main() -> int: return _tried_again(lambda: _streamed(args.argv)).returncode if args.command == "exists": return _exists(args.ref) + if args.command == "wanted": + return _wanted(args.entry) + if args.command == "build": + return _build(args.entry, args.arch) + if args.command == "config": + print(_config(args.image)) + return 0 if args.command == "smoke": - return _smoke(args.name, args.image) + if args.name in DISK_ARGS and args.launcher is None: + parser.error(f"{args.name} is booted in a QEMU launcher, which --launcher names") + return _smoke(args.name, args.image, args.launcher) + if args.command == "publish": + return _publish(args.missing) problems = _check(args.entry) if args.command == "check" else _verify_record(args.path) for problem in problems: print(f"::error::{problem}") return 1 if problems else 0 +def _wanted(entry: str) -> int: + if not _checked(entry): + return 1 + _sign_in() + ref = _ref(json.loads(entry)) + if (code := _exists(ref)) == 2: + return 1 + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"build={'true' if code == 1 else 'false'}\n") + if code == 0: + print(f"{ref} is published already, so it isn't built again") + return 0 + + +def _build(entry: str, arch: str) -> int: + """Build ENTRY for linux/``arch``, with no provenance attestation, which would show in the multi-platform image as an + unknown platform.""" + if not _checked(entry): + return 1 + image, target, build_args = json.loads(entry), f"linux/{arch}", [] + if image["name"] in DISK_ARGS: + args, rootfs = image["args"], f"{image['args']['ROOTFS']}-{arch}" + if code := _tried_again(lambda: _streamed(["docker", "pull", "--platform", target, rootfs])).returncode: + return code + for arg, value in {"ROOTFS": rootfs, "TOOLS": args["TOOLS"], "SIZE_GB": args["SIZE_GB"], + "ROOTFS_CONFIG": _config(rootfs)}.items(): + build_args += ["--build-arg", f"{arg}={value}"] + command = ["docker", "build", "--platform", target, "--provenance=false", "-f", image["dockerfile"], + "-t", f"{_ref(image)}-{arch}", *build_args, + "--label", f"org.opencontainers.image.source=https://github.com/{os.environ['GITHUB_REPOSITORY']}", + "--label", f"org.opencontainers.image.revision={os.environ['GITHUB_SHA']}", + "--label", f"io.agentenv.source-tag={image['tag']}", image["context"]] + return _tried_again(lambda: _streamed(command)).returncode + + +def _publish(missing: str) -> int: + _sign_in() + for image in json.loads(missing): + if not _checked(json.dumps(image)): + return 1 + ref = _ref(image) + if (code := _exists(ref)) == 0: + print(f"{ref} is published already, so it's left as it is") + continue + if code != 1: + return 1 + created = _tried_again(lambda: _streamed([ + "docker", "buildx", "imagetools", "create", "-t", ref, + "--annotation", f"index:org.opencontainers.image.source=https://github.com/{os.environ['GITHUB_REPOSITORY']}", + "--annotation", f"index:org.opencontainers.image.revision={os.environ['GITHUB_SHA']}", + f"{ref}-amd64", f"{ref}-arm64"])) + if created.returncode: + return created.returncode + return 0 + + +def _checked(entry: str) -> bool: + problems = _check(entry) + for problem in problems: + print(f"::error::{problem}") + return not problems + + +def _ref(image: dict) -> str: + return f"{image['repository']}:{image['tag']}" + + +def _sign_in() -> None: + """Sign in to GHCR with the job's token, as the workflow's actor.""" + subprocess.run(["docker", "login", "ghcr.io", "--username", os.environ["GITHUB_ACTOR"], "--password-stdin"], + input=os.environ["GHCR_TOKEN"], text=True, check=True) + + def _check(entry: str) -> list[str]: try: image = json.loads(entry) @@ -134,11 +254,35 @@ def _check(entry: str) -> list[str]: for key, value in expected.items() if image.get(key) != value] if not _TAG.match(str(image.get("tag"))): problems.append(f"{name}: {image.get('tag')!r} isn't a source tag") - if extra := sorted(set(image) - {*expected, "tag"}): + if name in DISK_ARGS: + problems += _check_args(name, image.get("args")) + if extra := sorted(set(image) - {*expected, "tag", *(("args",) if name in DISK_ARGS else ())}): problems.append(f"{name}: unexpected {', '.join(extra)}") return problems +def _check_args(name: str, args: object) -> list[str]: + """The problems with a disk form's build args: each image arg must name the image agent-env ships it's for, by a + source tag, and SIZE_GB must be its base's size.""" + wanted = DISK_ARGS[name] + if not isinstance(args, dict) or set(args) != {*wanted, "SIZE_GB"}: + return [f"{name}: args is {args!r}, not {', '.join(sorted({*wanted, 'SIZE_GB'}))}"] + problems = [] + for arg, image in wanted.items(): + repository, _, tag = str(args[arg]).rpartition(":") + if repository != f"{REGISTRY}/{image}" or not _TAG.match(tag): + problems.append(f"{name}: {arg} is {args[arg]!r}, not {REGISTRY}/{image} under a source tag") + if args["SIZE_GB"] != (size := str(_disk_size(name))): + problems.append(f"{name}: SIZE_GB is {args['SIZE_GB']!r}, not its base's min_disk_gb, {size}") + return problems + + +def _disk_size(name: str) -> int: + """The min_disk_gb of the base computer a disk form is made of, as its artifact.toml declares it.""" + toml = Path(IMAGES[DISK_ARGS[name]["ROOTFS"]]).with_name("artifact.toml") + return tomllib.loads((REPO / toml).read_text())["forms"]["disk"]["min_disk_gb"] + + def _tried_again(attempt: Callable[[], subprocess.CompletedProcess]) -> subprocess.CompletedProcess: """``attempt()``, and again after each of ``_RETRY_WAITS_SECONDS`` while what it printed says a later try can work (``_why_again``).""" @@ -232,9 +376,18 @@ def _verify_record(path: Path) -> list[str]: return problems -def _smoke(name: str, image: str) -> int: +def _config(image: str) -> str: + """The config object of ``image``'s OCI config, a local image's, as base64'd JSON.""" + inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", image], + capture_output=True, text=True, check=True) + return base64.b64encode(inspected.stdout.strip().encode()).decode() + + +def _smoke(name: str, image: str, launcher: str | None) -> int: if name == "agentenv-service-db": return _healthy(image) + if name in DISK_ARGS: + return _boots(name, image, launcher) command = [part.replace("{image}", image) for part in _SMOKES[name]] result = subprocess.run(command, capture_output=True, text=True) if result.returncode != 0: @@ -242,6 +395,36 @@ def _smoke(name: str, image: str) -> int: return result.returncode +def _boots(name: str, image: str, launcher: str) -> int: + """Boot ``image``, a build of the disk form ``name``, in ``launcher``, with KVM where it can, and wait for every + program its base computer's supervisord.conf declares to run, which its console shows.""" + programs = set(_PROGRAM.findall((REPO / Path(IMAGES[DISK_ARGS[name]["ROOTFS"]]).with_name("supervisord.conf")) + .read_text())) + arch = subprocess.run(["docker", "image", "inspect", "--format", "{{.Architecture}}", image], capture_output=True, + text=True, check=True).stdout.strip() + machine = f"{image}-machine" + subprocess.run(["docker", "build", "-q", "-t", machine, "-"], text=True, check=True, capture_output=True, + input=f"FROM --platform=linux/{arch} {image} AS disk\nFROM {launcher}\nCOPY --from=disk /disk/ /disk/\n") + kvm = ["--device", "/dev/kvm"] if Path("/dev/kvm").exists() else [] + container = subprocess.run(["docker", "run", "-d", *kvm, "-e", f"VM_ARCH={arch}", machine], capture_output=True, + text=True, check=True).stdout.strip() + try: + deadline, logs = time.monotonic() + _BOOTED_WITHIN_SECONDS, "" + while time.monotonic() < deadline: + logs = subprocess.run(["docker", "logs", container], capture_output=True, text=True, errors="replace") + logs = logs.stdout + logs.stderr + if programs <= set(_RUNNING.findall(logs)): + native = {"x86_64": "amd64", "aarch64": "arm64"}.get(platform.machine()) == arch + print(f"{image} booted to its desktop {'with KVM' if kvm and native else 'under TCG'}") + return 0 + time.sleep(_HEALTH_POLL_SECONDS) + tail = "\n".join(logs.splitlines()[-_LOG_TAIL_LINES:]) + print(f"::error::{image} didn't boot to its desktop within {_BOOTED_WITHIN_SECONDS}s:\n{tail}") + return 1 + finally: + subprocess.run(["docker", "rm", "-f", container], capture_output=True) + + def _healthy(image: str) -> int: """Start ``image`` and wait for its health check to pass, as compose does before starting what depends on it.""" container = subprocess.run(["docker", "run", "-d", image], capture_output=True, text=True, check=True).stdout.strip() diff --git a/.github/scripts/infra_images.py b/.github/scripts/infra_images.py index 6c520cade..8052fe044 100644 --- a/.github/scripts/infra_images.py +++ b/.github/scripts/infra_images.py @@ -2,11 +2,13 @@ the release workflow's jobs that only read: this imports agent-env, so it runs where agent-env and its dependencies are installed, and no job that holds a token to push images or publish to PyPI runs it (those run ``infra_build.py``). -``plan`` prints each image's name, repository, tag, Dockerfile and context as JSON, ``--missing`` only those the +``plan`` prints each image's name, repository, tag, Dockerfile and context as JSON, and for an image built over other +published images, its build args, each naming one by its repository and tag or giving a value, ``--missing`` only those the registry hasn't published: it has no image under the tag, or refuses to show one, as GHCR does for a package that -doesn't exist yet, so the first publish can start. ``--only infra`` lists only the infra images, and ``--only agents`` +doesn't exist yet, so the first publish can start. ``--only infra`` lists only the infra images, ``--only agents`` only the images that install the protocol, the agents' and the website browser's, which a release builds after it -publishes the protocol, as their protocol floor may name it. ``write PATH`` records the digest the registry serves under every +publishes the protocol, as their protocol floor may name it, and ``--only disks`` the base computers' disk forms, built +over their published images. ``write PATH`` records the digest the registry serves under every image's tag as a release wheel carries them, and fails if any is missing. ``public`` warns about each image a user, who pulls anonymously, can't. The registry is read with ``GHCR_TOKEN`` (as ``GITHUB_ACTOR``) when it's set, anonymously otherwise, and always anonymously by ``public``. @@ -21,7 +23,7 @@ from pathlib import Path from agent_env.env.bootstrap import PLAYWRIGHT_MCP_DOCKERFILE, PUBLISHED_IMAGES -from agent_env.env.published_images import BUNDLE_IMAGES, REGISTRY, PublishedImage, write_release_digests +from agent_env.env.published_images import BUNDLE_IMAGES, DISK_IMAGES, REGISTRY, PublishedImage, write_release_digests from agent_env.store.image_store.oci_registry_credentials import RegistryAuth from agent_env.store.image_store.registry_api import NoSuchImageError, RegistryRefusedError, pin_digest @@ -30,7 +32,8 @@ # example env's, and the Playwright MCP browser's, whose recorder installs it, matched by its Dockerfile. _AFTER_PROTOCOL = BUNDLE_IMAGES + tuple(image for image in PUBLISHED_IMAGES if image.dockerfile == PLAYWRIGHT_MCP_DOCKERFILE) -_GROUPS = {"infra": tuple(image for image in PUBLISHED_IMAGES if image not in _AFTER_PROTOCOL), "agents": _AFTER_PROTOCOL} +_GROUPS = {"infra": tuple(image for image in PUBLISHED_IMAGES if image not in _AFTER_PROTOCOL + DISK_IMAGES), + "agents": _AFTER_PROTOCOL, "disks": DISK_IMAGES} def main() -> int: @@ -59,9 +62,13 @@ def main() -> int: return 0 -def _described(image: PublishedImage) -> dict[str, str]: - return {"name": image.name, "repository": image.repository, "tag": image.tag(), - "dockerfile": str(image.dockerfile.relative_to(REPO)), "context": str(image.dockerfile.parent.relative_to(REPO))} +def _described(image: PublishedImage) -> dict: + described = {"name": image.name, "repository": image.repository, "tag": image.tag(), + "dockerfile": str(image.dockerfile.relative_to(REPO)), + "context": str(image.dockerfile.parent.relative_to(REPO))} + if image.args or image.build_args: + described["args"] = {**{arg: built.ref() for arg, built in image.args}, **dict(image.build_args)} + return described def _digest(image: PublishedImage) -> str | None: diff --git a/.github/workflows/local-backends.yml b/.github/workflows/local-backends.yml index 0b68f8a6a..1ae3b2285 100644 --- a/.github/workflows/local-backends.yml +++ b/.github/workflows/local-backends.yml @@ -138,6 +138,9 @@ jobs: run: | docker run --detach --name registry --publish 5000:5000 mirror.gcr.io/library/registry:2 timeout 30 sh -c 'until curl -sf http://localhost:5000/v2/ >/dev/null; do sleep 1; done' + # x64 hosted runners have /dev/kvm; the disk form's tests boot under it, and skip without it, which fails below. + - name: Let the local provider boot disks with KVM + run: sudo chmod 666 /dev/kvm - name: Slow integration tier on the local defaults env: AGENTENV_TEST_SHARD: ${{ matrix.shard }}/8 diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 752c5c71b..a3b59d054 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -9,11 +9,12 @@ name: Publish to PyPI # rerun is safe. # # It also publishes the images agent-env ships (agent_env.env.published_images), its infra images and its bundles' -# (its agents' and the mail example's env), +# (its agents' and the mail example's env), and its base computers' disk forms, # whose tags the registry lacks, and records their digests in the wheel and the release notes, once the repository # variable PUBLISH_INFRA_IMAGES is "true". The infra images are published before agentenv-framework-protocol. The # bundles' images keep a protocol floor, so each is built only when its folder changes, once the protocol is on PyPI, -# as a raised floor may name the protocol this release ships. agentenv-framework waits for every image. Every job that +# as a raised floor may name the protocol this release ships. A disk form is built over its base computer's published +# image with the published disk tools, so after both. agentenv-framework waits for every image. Every job that # reads the shipped bundles' folders first pins the protocol in their envs and artifacts as the wheel does # (.github/scripts/shipped_bundles.py pin), so an image's tag, its build and the wheel all see the same sources. Run # by hand on main, the workflow publishes the images alone, with nothing sent to PyPI; an agent's image then installs @@ -71,6 +72,7 @@ jobs: outputs: missing_infra: ${{ steps.plan.outputs.missing_infra }} missing_agents: ${{ steps.plan.outputs.missing_agents }} + missing_disks: ${{ steps.plan.outputs.missing_disks }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -90,8 +92,10 @@ jobs: uv sync --locked --no-dev missing_infra=$(uv run --no-sync python .github/scripts/infra_images.py plan --missing --only infra) missing_agents=$(uv run --no-sync python .github/scripts/infra_images.py plan --missing --only agents) + missing_disks=$(uv run --no-sync python .github/scripts/infra_images.py plan --missing --only disks) echo "missing_infra=$missing_infra" >> "$GITHUB_OUTPUT" echo "missing_agents=$missing_agents" >> "$GITHUB_OUTPUT" + echo "missing_disks=$missing_disks" >> "$GITHUB_OUTPUT" # Once main's CI has passed, so nothing is pushed for a commit that fails it: builds each missing infra image on a # runner of each platform, checks that it runs, and pushes it under its tag and platform (src--amd64), which @@ -116,11 +120,8 @@ jobs: ENTRY: ${{ toJSON(matrix.image) }} NAME: ${{ matrix.image.name }} DOCKERFILE: ${{ matrix.image.dockerfile }} - CONTEXT: ${{ matrix.image.context }} - TAG: ${{ matrix.image.tag }} - REF: ${{ matrix.image.repository }}:${{ matrix.image.tag }} + ARCH: ${{ matrix.arch }} BUILT: ${{ matrix.image.repository }}:${{ matrix.image.tag }}-${{ matrix.arch }} - PLATFORM: linux/${{ matrix.arch }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -131,28 +132,13 @@ jobs: id: wanted env: GHCR_TOKEN: ${{ github.token }} - run: | - python3 .github/scripts/infra_build.py check "$ENTRY" - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - code=0 - python3 .github/scripts/infra_build.py exists "$REF" || code=$? - case "$code" in - 0) echo "build=false" >> "$GITHUB_OUTPUT"; echo "$REF is published already, so it isn't built again" ;; - 1) echo "build=true" >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac + run: python3 .github/scripts/infra_build.py wanted "$ENTRY" - name: Pull its base images from mirror.gcr.io, as public.ecr.aws can rate-limit a runner for hours if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/mirror_base_images.py "$DOCKERFILE" - name: Build if: steps.wanted.outputs.build == 'true' - run: | - # No provenance attestation, which would show in the multi-platform image as an unknown platform. - python3 .github/scripts/infra_build.py retry \ - docker build --platform "$PLATFORM" --provenance=false -f "$DOCKERFILE" -t "$BUILT" \ - --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ - --label "io.agentenv.source-tag=${TAG}" "$CONTEXT" + run: python3 .github/scripts/infra_build.py build "$ENTRY" "$ARCH" - name: Check that it runs if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/infra_build.py smoke "$NAME" "$BUILT" @@ -180,23 +166,7 @@ jobs: - name: Tag each image's platform builds as one multi-platform image under its source tag env: GHCR_TOKEN: ${{ github.token }} - run: | - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - jq -c '.[]' <<< "$MISSING" | while read -r entry; do - python3 .github/scripts/infra_build.py check "$entry" - ref=$(jq -r '"\(.repository):\(.tag)"' <<< "$entry") - code=0 - python3 .github/scripts/infra_build.py exists "$ref" || code=$? - if [ "$code" = 0 ]; then - echo "$ref is published already, so it's left as it is" - continue - fi - [ "$code" = 1 ] || exit 1 - python3 .github/scripts/infra_build.py retry docker buildx imagetools create -t "$ref" \ - --annotation "index:org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --annotation "index:org.opencontainers.image.revision=${GITHUB_SHA}" \ - "${ref}-amd64" "${ref}-arm64" - done + run: python3 .github/scripts/infra_build.py publish "$MISSING" # When images are published, the protocol waits for the infra images, so a failed infra image job sends neither # package to PyPI. A failed agent image job, which runs after it, keeps agentenv-framework off PyPI. @@ -261,11 +231,8 @@ jobs: ENTRY: ${{ toJSON(matrix.image) }} NAME: ${{ matrix.image.name }} DOCKERFILE: ${{ matrix.image.dockerfile }} - CONTEXT: ${{ matrix.image.context }} - TAG: ${{ matrix.image.tag }} - REF: ${{ matrix.image.repository }}:${{ matrix.image.tag }} + ARCH: ${{ matrix.arch }} BUILT: ${{ matrix.image.repository }}:${{ matrix.image.tag }}-${{ matrix.arch }} - PLATFORM: linux/${{ matrix.arch }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -276,28 +243,13 @@ jobs: id: wanted env: GHCR_TOKEN: ${{ github.token }} - run: | - python3 .github/scripts/infra_build.py check "$ENTRY" - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - code=0 - python3 .github/scripts/infra_build.py exists "$REF" || code=$? - case "$code" in - 0) echo "build=false" >> "$GITHUB_OUTPUT"; echo "$REF is published already, so it isn't built again" ;; - 1) echo "build=true" >> "$GITHUB_OUTPUT" ;; - *) exit 1 ;; - esac + run: python3 .github/scripts/infra_build.py wanted "$ENTRY" - name: Pull its base images from mirror.gcr.io, as public.ecr.aws can rate-limit a runner for hours if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/mirror_base_images.py "$DOCKERFILE" - name: Build if: steps.wanted.outputs.build == 'true' - run: | - # No provenance attestation, which would show in the multi-platform image as an unknown platform. - python3 .github/scripts/infra_build.py retry \ - docker build --platform "$PLATFORM" --provenance=false -f "$DOCKERFILE" -t "$BUILT" \ - --label "org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ - --label "io.agentenv.source-tag=${TAG}" "$CONTEXT" + run: python3 .github/scripts/infra_build.py build "$ENTRY" "$ARCH" - name: Check that it runs if: steps.wanted.outputs.build == 'true' run: python3 .github/scripts/infra_build.py smoke "$NAME" "$BUILT" @@ -325,35 +277,102 @@ jobs: - name: Tag each image's platform builds as one multi-platform image under its source tag env: GHCR_TOKEN: ${{ github.token }} + run: python3 .github/scripts/infra_build.py publish "$MISSING" + + # Each base computer's disk form whose tag the registry lacks, built on a runner of each platform from the base's + # published image of that platform and the published disk tools, booted with the QEMU launcher built from this + # checkout (with KVM on x64 runners, under TCG where a runner has none, arm64's), and pushed under its tag and + # platform as infra-build does; disk-publish then tags the two as one multi-platform image. + disk-build: + needs: [main-ci, infra-plan, infra-publish, agent-publish] + if: >- + !cancelled() && needs.main-ci.result == 'success' && needs.infra-plan.result == 'success' + && needs.infra-plan.outputs.missing_disks != '[]' + && contains(fromJSON('["success", "skipped"]'), needs.infra-publish.result) + && contains(fromJSON('["success", "skipped"]'), needs.agent-publish.result) + strategy: + fail-fast: true + matrix: + image: ${{ fromJSON(needs.infra-plan.outputs.missing_disks) }} + arch: [amd64, arm64] + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + timeout-minutes: 60 + permissions: + contents: read + packages: write + env: + ENTRY: ${{ toJSON(matrix.image) }} + NAME: ${{ matrix.image.name }} + DOCKERFILE: ${{ matrix.image.dockerfile }} + ARCH: ${{ matrix.arch }} + BUILT: ${{ matrix.image.repository }}:${{ matrix.image.tag }}-${{ matrix.arch }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - name: Pin agentenv-framework-protocol in the shipped bundles, as the release does + run: python3 .github/scripts/shipped_bundles.py pin + - name: Check that it's an image agent-env ships, and that no run has published it + id: wanted + env: + GHCR_TOKEN: ${{ github.token }} + run: python3 .github/scripts/infra_build.py wanted "$ENTRY" + - name: Pull its base images, and the launcher's, from mirror.gcr.io, as public.ecr.aws can rate-limit a runner + if: steps.wanted.outputs.build == 'true' + run: python3 .github/scripts/mirror_base_images.py "$DOCKERFILE" src/agent_env/computers/disk/linux/launcher/Dockerfile + - name: Let the launcher use KVM, where the runner has it + if: steps.wanted.outputs.build == 'true' run: | - echo "$GHCR_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin - jq -c '.[]' <<< "$MISSING" | while read -r entry; do - python3 .github/scripts/infra_build.py check "$entry" - ref=$(jq -r '"\(.repository):\(.tag)"' <<< "$entry") - code=0 - python3 .github/scripts/infra_build.py exists "$ref" || code=$? - if [ "$code" = 0 ]; then - echo "$ref is published already, so it's left as it is" - continue - fi - [ "$code" = 1 ] || exit 1 - python3 .github/scripts/infra_build.py retry docker buildx imagetools create -t "$ref" \ - --annotation "index:org.opencontainers.image.source=https://github.com/${GITHUB_REPOSITORY}" \ - --annotation "index:org.opencontainers.image.revision=${GITHUB_SHA}" \ - "${ref}-amd64" "${ref}-arm64" - done + if [ -e /dev/kvm ]; then + sudo chmod 666 /dev/kvm + fi + - name: Build + if: steps.wanted.outputs.build == 'true' + run: python3 .github/scripts/infra_build.py build "$ENTRY" "$ARCH" + - name: Build the QEMU launcher from this checkout, to boot it in + if: steps.wanted.outputs.build == 'true' + run: docker build -q -t agentenv-qemu-launcher src/agent_env/computers/disk/linux/launcher + - name: Check that it boots + if: steps.wanted.outputs.build == 'true' + run: python3 .github/scripts/infra_build.py smoke "$NAME" "$BUILT" --launcher agentenv-qemu-launcher + - name: Push it under its tag and platform + if: steps.wanted.outputs.build == 'true' + run: python3 .github/scripts/infra_build.py retry docker push "$BUILT" + + # Each new disk form's platform builds tagged as one multi-platform image, as infra-publish does the infra images. + disk-publish: + needs: [main-ci, infra-plan, disk-build] + if: >- + !cancelled() && needs.main-ci.result == 'success' && needs.disk-build.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: write + env: + MISSING: ${{ needs.infra-plan.outputs.missing_disks }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + - name: Tag each image's platform builds as one multi-platform image under its source tag + env: + GHCR_TOKEN: ${{ github.token }} + run: python3 .github/scripts/infra_build.py publish "$MISSING" # The digest the registry serves under every image's tag, for the wheel and the release notes, which the framework # job checks against the registry before the wheel carries it; and a warning for each image an anonymous pull, as an # install makes, doesn't get: GHCR keeps a new package private until its admin makes it public. infra-record: - needs: [infra-plan, infra-build, infra-publish, agent-build, agent-publish] + needs: [infra-plan, infra-build, infra-publish, agent-build, agent-publish, disk-build, disk-publish] if: >- !cancelled() && needs.infra-plan.result == 'success' && contains(fromJSON('["success", "skipped"]'), needs.infra-build.result) && contains(fromJSON('["success", "skipped"]'), needs.infra-publish.result) && contains(fromJSON('["success", "skipped"]'), needs.agent-build.result) && contains(fromJSON('["success", "skipped"]'), needs.agent-publish.result) + && contains(fromJSON('["success", "skipped"]'), needs.disk-build.result) + && contains(fromJSON('["success", "skipped"]'), needs.disk-publish.result) runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/AGENTS.md b/AGENTS.md index d22df7f7b..0fd951ba4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,7 +116,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | @@ -132,7 +132,7 @@ and deserialized through a registry. | `explorer/` | Optional local web UI: `agent-env up`, binds loopback `:8234`. | | `cli/` | Click CLI with the groups `a2a-agent`, `artifact`, `bundle`, `config`, `env`, `eval`, `plugin`, `run`, `task`, `up`. `task run`, `task run-batch` and `eval run` hand their runs to the configured `[runner]` and follow them. | | `agents/` | The `agents` bundle: the agents agent-env ships, one folder each under `agents/agents/`, with a smoke task each under `agents/tasks/`. Its Claude Code, `@local/agentenv-framework/agents/claude-code`, is the default agent. An agent's Python packages are its folder's `requirements.txt`, installed in its image, never in agent-env's own environment. A run that pulls one of its agents reads the whole bundle, so `tst/unit/bundle/shipped_bundles_test.py` parses and checks every shipped bundle. | -| `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. | +| `computers/` | The `computers` bundle: the base computers agent-env ships as `computer` artifacts, `ubuntu-24.04` (XFCE on Xvfb, supervisord as the session user, settings as `AGENTENV_SETTING_`), aliased through `[artifacts]`. It has no tasks either. `disk/linux/` is the Linux disk form's recipe: `boot/`, whose Dockerfile makes a disk form (a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs whose init reads a NoCloud seed and `switch_root`s into the image's entrypoint, and an ext4 root filesystem that is a container image's, plus `/etc/agentenv/vm/image.env` from its OCI config), `tools/`, the disk tools that make it unprivileged on the builder's own platform (`mkfs.ext4 -d`), and `launcher/`, the QEMU launcher that boots one under KVM. A base's `[forms.disk]` with no image is made from its container form; the composer makes each arch's disk from its composed container, keeping the base disk's EFI system partition, keyed by the container's key, the base disk and the recipe; gate 3 probes it where a provider boots disks natively (`disk_image_platforms()`), and a deploy's first boot with KVM probes one left `unverified`; a deploy naming `form = "disk"` boots it through `create_vm(image=..., env=...)`, its settings in the seed. | | `browsers/` | The `browsers` bundle: the website browsers agent-env ships, today `chromium`, a `computer_app` env (Chrome for Testing at the build Playwright pins, for amd64 and arm64, set up for unattended use by managed policies in `/etc/opt/chrome_for_testing/policies/managed`, opened at boot on its `start_urls`, its profile its data). All of it, its pin, install step, policies and command, is in `envs/chromium/chromium.py`; `tst/integration/computer/chromium_app_test.py` runs it. Like `drivers`, it has no tasks. | | `drivers/` | The `drivers` bundle: the computer drivers agent-env ships as `computer_driver` envs, the reference driver (X11 on Linux, recording through `x11grab`) aliased `reference`, and trycua's cua-driver aliased `cua-driver` (its daemon from the pinned PyPI wheel, its own instructions, tools and skill resources served to agents through `@server_instructions`, `forwarded_tools` and `@resource`, its tools beside `computer_*`, `computer/v1` on its desktop-scope calls, its skill pack leading the computer skill; `tst/integration/computer/cua_driver_test.py` runs it). It has no tasks: a bundle with neither tasks nor evals is refused unless its `aliases.toml` names something for other bundles to use. The shipped `desktop` example composes it with the `ubuntu-24.04` base into a computer env, which `tst/integration/computer/` deploys, probes, records and has an agent use. | | `examples/` | The example bundles agent-env ships (`hello`, and `echo-mcp`, which needs Docker, and `desktop`, a computer env of `ubuntu-24.04` and `reference` with a probe task and an agent task). Tests run them, but they are examples for users, not fixtures. Every shipped bundle is registered under `agent_env.bundles` in `pyproject.toml`, ` = ""` shipping `src///`. | @@ -302,7 +302,7 @@ orchestrators call it the same way. What callers rely on: - Skips are declared capability gaps, decided at collection time: the gates in `tst/util/capabilities.py` produce `skipif` marks with the reason `agentenv-capability-missing: ` (`model_endpoint_configured`, `remote_sandbox`, - `default_a2a_agent`, `mcp_server_sources`, `gnu_stat`, `trl_installed`). CI rejects any other skip reason; a broken configuration must fail loudly, not skip. + `default_a2a_agent`, `mcp_server_sources`, `gnu_stat`, `trl_installed`, `kvm`). CI rejects any other skip reason; a broken configuration must fail loudly, not skip. - The suite is backend-agnostic: a test about one particular backend lives with that backend, not here. A custom store must pass `tst/store/{conformance,object_conformance,image_conformance,secret_conformance}.py`, the same kits the built-ins pass, and a custom `[runner]` must pass `tst/runner/conformance.py`, which @@ -372,7 +372,10 @@ and creates the GitHub release, whose notes are the titles of the pull requests merged since the previous release. Once the repository variable `PUBLISH_INFRA_IMAGES` is `true`, it also publishes the infra images agent-env ships and the shipped bundles', one per agent folder, one for the env `a2a-agent validate` deploys (the `mail-mcp` -example's) and one per shipped base computer (`agentenv-ubuntu-24.04`; `agent_env.env.published_images`), whose `src-` tags GHCR lacks: after `main`'s CI passes, each is +example's) and one per shipped base computer (`agentenv-ubuntu-24.04`; `agent_env.env.published_images`), the disk tools +and QEMU launcher, and each base computer's disk form (`agentenv-ubuntu-24.04-disk`, tagged by the boot recipe's sources +and its base's and the tools' tags, built over them once they're published and booted in the launcher, with KVM on +x64 runners and under TCG on arm64's), whose `src-` tags GHCR lacks: after `main`'s CI passes, each is built on a runner of each platform, checked, and tagged as one multi-platform image, and a tag that exists is never moved. The infra images are published before the protocol, and a failed infra image job sends neither package to PyPI. An agent's `requirements.txt`, and that env's, keeps its protocol floor, unpinned, so its image is built again diff --git a/src/agent_env/artifact/artifacts/computer.py b/src/agent_env/artifact/artifacts/computer.py index 98858bfc9..61d5cc773 100644 --- a/src/agent_env/artifact/artifacts/computer.py +++ b/src/agent_env/artifact/artifacts/computer.py @@ -5,25 +5,39 @@ from __future__ import annotations +import functools import re +from collections.abc import Callable from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self -from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializerFunctionWrapHandler, + ValidationError, + field_validator, + model_serializer, + model_validator, +) from agent_env.artifact.artifact import Artifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.ref import ArtifactRef +from agent_env.config import get_config from agent_env.entity.refs import EntityRef from agent_env.store.base import NotFoundError -from agent_env.store.image_store.oci_registry_credentials import names_registry -from agent_env.store.image_store.image_contents import UnreadableLayerError, os_release, platform_manifests +from agent_env.store.image_store.oci_registry_credentials import RegistryAuth, names_registry +from agent_env.store.image_store.image_contents import UnreadableLayerError, disk_layer, os_release, platform_manifests if TYPE_CHECKING: from agent_env.bundle.authoring import AuthoringContext OS_TYPES = ("linux",) -FORMS = ("container",) +FORMS = ("container", "disk") +#: The file a disk form's image holds, as KubeVirt's containerDisk has it, owned by the uid its QEMU runs as. +DISK_FILE, DISK_OWNER = "disk/disk.img", 107 #: What a setting holds: its default's type, which a value an env or a deploy gives must share. SettingValue = str | int | float | bool @@ -81,10 +95,30 @@ class ContainerForm(BaseModel): input_channel: Literal["env"] +class DiskForm(BaseModel): + """The computer as a disk a VM boots from UEFI: an EFI system partition holding one UKI, whose init reads deploy + settings from a NoCloud seed (``input_channel``) and starts the image's own ``init``, and its root filesystem. With + no ``image``, it's the container form's root filesystem on agent-env's Linux boot recipe, made when composed.""" + + model_config = ConfigDict(extra="forbid") + + image: ArtifactRef | None = None + init: Literal["supervisord"] + input_channel: Literal["nocloud"] + firmware: list[Literal["efi"]] = Field(default=["efi"], min_length=1) + min_disk_gb: int = Field(ge=1, description="Its virtual size, which a deploy's disk is at least") + + class Forms(BaseModel): model_config = ConfigDict(extra="forbid") container: ContainerForm + disk: DiskForm | None = None + + @model_serializer(mode="wrap") + def _without_absent_forms(self, handler: SerializerFunctionWrapHandler) -> dict[str, Any]: + """Only the forms the computer has, so a computer of one form is stored as it was before others existed.""" + return {form: value for form, value in handler(self).items() if value is not None} class ImageCheck(BaseModel): @@ -115,8 +149,9 @@ class ComputerArtifact(Artifact): forms: Forms config: dict[str, Setting] = Field(default_factory=dict, description="Its settings, by name") - toml_refs: ClassVar[tuple[EntityRef, ...]] = (EntityRef.artifact("forms.container.image", - artifact_type="docker_image"),) + toml_refs: ClassVar[tuple[EntityRef, ...]] = ( + EntityRef.artifact("forms.container.image", artifact_type="docker_image"), + EntityRef.artifact("forms.disk.image", artifact_type="docker_image")) toml_keys: ClassVar[dict[str, type]] = {"description": str, "arch": list, "os": dict, "session": dict, "forms": dict, "config": dict} @@ -158,9 +193,15 @@ def put(cls, *, inspect: bool = True, **fields: Any) -> Self: from agent_env.artifact.store import get_artifact_store computer = cls(**fields) - image = computer.image() - checks = {"container": _inspect_container(image, computer.arch, computer.os) if inspect - else _unread(computer.arch, "inspection was skipped")} + image, disk = computer.image(), computer.forms.disk + checks = {"container": _inspect(image, computer.arch, functools.partial(_read_container, computer.os)) + if inspect else _unread(computer.arch, "inspection was skipped")} + if disk is not None and disk.image is None: + checks["disk"] = _unread(computer.arch, "it's made from the container form when composed") + elif disk is not None: + checks["disk"] = (_inspect(computer.disk_image(), computer.arch, + functools.partial(_read_disk, disk.min_disk_gb)) + if inspect else _unread(computer.arch, "inspection was skipped")) problems = [f"forms.{form} ({arch}): {check.problem}" for form, by_arch in checks.items() for arch, check in by_arch.items() if check.status == "failed"] if problems: @@ -175,10 +216,10 @@ def from_toml(cls, data: dict[str, Any], ctx: AuthoringContext) -> Self: """Write the computer an artifact.toml declares, its form's image as the bundle wrote it or the artifact the key names.""" fields = {key: value for key, value in data.items() if key not in ("id", "type", "version")} - container = (fields.get("forms") or {}).get("container") - if isinstance(container, dict) and "image" in container: - image = ctx.artifact(container["image"]) - container["image"] = {"id": image.id, "version": image.version} + for form in (fields.get("forms") or {}).values(): + if isinstance(form, dict) and "image" in form: + image = ctx.artifact(form["image"]) + form["image"] = {"id": image.id, "version": image.version} try: return cls.put(id=ctx.id, **fields) except ValidationError as e: @@ -189,16 +230,25 @@ def from_toml(cls, data: dict[str, Any], ctx: AuthoringContext) -> Self: def image(self) -> DockerImageArtifact: """The container form's image artifact, refused when the store has no docker_image artifact of that id and version.""" - ref = self.forms.container.image + return self._image("container", self.forms.container.image) + + def disk_image(self) -> DockerImageArtifact | None: + """The disk form's image artifact, None when it has none, refused as ``image`` is.""" + disk = self.forms.disk + return None if disk is None or disk.image is None else self._image("disk", disk.image) + + def _image(self, form: str, ref: ArtifactRef) -> DockerImageArtifact: try: return DockerImageArtifact.get(ref.id, ref.version) except (NotFoundError, TypeError) as e: - raise ValueError(f"computer {self.id!r}: forms.container.image: {e}") from e - + raise ValueError(f"computer {self.id!r}: forms.{form}.image: {e}") from e -def _inspect_container(image: DockerImageArtifact, arches: list[str], os: LinuxOS) -> dict[str, ImageCheck]: - from agent_env.config import get_config +def _inspect(image: DockerImageArtifact, arches: list[str], + read: Callable[[str, str, RegistryAuth | None], ImageCheck]) -> dict[str, ImageCheck]: + """Each arch's image of a form, ``read`` from its registry without pulling it: failed when the registry serves none + for an arch or ``read`` refuses it, uninspected when this Python can't read its layers, so its first boot checks + it.""" if not names_registry(image.image_name): how = "only its build context is stored" if image.context_only else "it's stored under a tag no registry serves" return _unread(arches, f"{how}, so its first boot checks it") @@ -215,23 +265,43 @@ def _inspect_container(image: DockerImageArtifact, arches: list[str], os: LinuxO checks[arch] = ImageCheck(status="failed", image=ref, problem=f"it has no linux/{arch} image, only {served}") continue try: - release = os_release(ref, digest, auth) + checks[arch] = read(ref, digest, auth) except UnreadableLayerError as e: checks[arch] = ImageCheck(status="uninspected", image=ref, manifest=digest, reason=f"{e}, so its first boot checks it") - continue except ValueError as e: - checks[arch] = ImageCheck(status="failed", image=ref, manifest=digest, - problem=f"its os-release couldn't be read: {e}") - continue - expected = {"ID": os.distribution, "VERSION_ID": None if os.version == "rolling" else os.version} - problems = [f"os-release has {key}={release.get(key)!r}, not {want!r}" for key, want in expected.items() - if release.get(key) != want] - checks[arch] = ImageCheck(status="failed" if problems else "passed", image=ref, manifest=digest, - os_release={key: release[key] for key in expected if key in release}, - problem="; ".join(problems) or None) + checks[arch] = ImageCheck(status="failed", image=ref, manifest=digest, problem=str(e)) return checks +def _read_container(os: LinuxOS, ref: str, digest: str, auth: RegistryAuth | None) -> ImageCheck: + """The container form's image manifest ``digest``, by its os-release against ``os``.""" + try: + release = os_release(ref, digest, auth) + except UnreadableLayerError: + raise + except ValueError as e: + raise ValueError(f"its os-release couldn't be read: {e}") from e + expected = {"ID": os.distribution, "VERSION_ID": None if os.version == "rolling" else os.version} + problems = [f"os-release has {key}={release.get(key)!r}, not {want!r}" for key, want in expected.items() + if release.get(key) != want] + return ImageCheck(status="failed" if problems else "passed", image=ref, manifest=digest, + os_release={key: release[key] for key in expected if key in release}, + problem="; ".join(problems) or None) + + +def _read_disk(min_disk_gb: int, ref: str, digest: str, auth: RegistryAuth | None) -> ImageCheck: + """The disk form's image manifest ``digest``, by the start of its one layer: its file, its owner and its virtual + size.""" + found = disk_layer(ref, digest, auth) + problems = [problem for problem, wrong in ( + (f"its first file is {found.path}, not {DISK_FILE}", found.path != DISK_FILE), + (f"{found.path} is owned by uid {found.uid}, not {DISK_OWNER}", found.uid != DISK_OWNER), + (f"its virtual size is {found.virtual_size / 2**30:.1f} GiB, under min_disk_gb ({min_disk_gb})", + found.virtual_size < min_disk_gb * 2**30)) if wrong] + return ImageCheck(status="failed" if problems else "passed", image=ref, manifest=digest, + problem="; ".join(problems) or None) + + def _unread(arches: list[str], why: str) -> dict[str, ImageCheck]: return {arch: ImageCheck(status="uninspected", reason=why) for arch in arches} diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 9c260dae0..6934bfd96 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -8,6 +8,18 @@ container form's image is built from the folder's `Dockerfile`, and a release pu building it. Name it from another bundle as `ubuntu-24.04`, the alias `aliases.toml` gives it, or by its id, `@local/agentenv-framework/computers/ubuntu-24.04`. +It also has a disk form, a disk a VM boots from UEFI, which is the container form's root filesystem on the Linux boot +recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs, +and an ext4 root filesystem that is the image's own, with `/etc/agentenv/vm/image.env`, its entrypoint, command, +user, working folder and environment, from its OCI config. The init mounts it, brings up the network, reads the deploy's +settings from a NoCloud seed (`CIDATA`) and becomes the image's entrypoint, as its user. A release publishes it as +`ghcr.io/scaleapi/agentenv-ubuntu-24.04-disk:src-`, tagged by both its sources; without it, composing makes it. +Composing a computer env makes each arch's disk from its composed container, the root filesystem rebuilt with +`mkfs.ext4 -d` by the disk tools (`disk/linux/tools/`) on the builder's own platform and the base disk's EFI system +partition kept, so nothing is emulated or privileged. A disk boots where `/dev/kvm` is, in the QEMU launcher +(`disk/linux/launcher/`): gate 3 probes it there, and elsewhere it stays `unverified` until a deploy's first boot with +KVM probes it. A deploy boots it with `env_params = { form = "disk" }`. + Its settings reach the computer as environment variables, `AGENTENV_SETTING_`, read as it starts: | Setting | Default | What it does | diff --git a/src/agent_env/computers/artifacts/ubuntu-24.04/artifact.toml b/src/agent_env/computers/artifacts/ubuntu-24.04/artifact.toml index 654f2f5e4..07f3f05ee 100644 --- a/src/agent_env/computers/artifacts/ubuntu-24.04/artifact.toml +++ b/src/agent_env/computers/artifacts/ubuntu-24.04/artifact.toml @@ -9,6 +9,12 @@ image = { dockerfile = "Dockerfile" } init = "supervisord" input_channel = "env" +[forms.disk] +init = "supervisord" +input_channel = "nocloud" +firmware = ["efi"] +min_disk_gb = 16 + [config.resolution] default = "1280x800" phase = "deploy" diff --git a/src/agent_env/computers/disk/linux/boot/Dockerfile b/src/agent_env/computers/disk/linux/boot/Dockerfile new file mode 100644 index 000000000..e91a97261 --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/Dockerfile @@ -0,0 +1,39 @@ +# check=skip=InvalidDefaultArgInFrom +# A Linux computer's disk form for one arch: a GPT disk (qcow2) of an EFI system partition holding one UKI, Ubuntu's +# kernel with a busybox initramfs whose init starts the computer, and an ext4 root filesystem that is ROOTFS, a container +# image's root filesystem, plus /etc/agentenv/vm/image.env, its start, from ROOTFS_CONFIG (its OCI config's config +# object, as base64'd JSON). The EFI system partition comes from ESP: the uki stage, on a disk of SIZE_GB, or a disk +# form's image, whose partition and size the disk keeps. Only the uki stage runs the target's programs: the disk itself +# is made on the build's own platform by TOOLS, the disk tools image, so an existing ESP needs no emulation. +ARG ROOTFS +ARG TOOLS +ARG ESP=uki + +FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c AS kernel +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + linux-image-virtual busybox-static kmod zstd cpio systemd-boot-efi systemd-ukify python3-pefile \ + && rm -rf /var/lib/apt/lists/* +COPY init udhcpc.script uki.sh /build/ +RUN sh /build/uki.sh /uki + +FROM scratch AS uki +COPY --from=kernel /uki/ / + +FROM --platform=$BUILDPLATFORM ${TOOLS} AS start +ARG ROOTFS_CONFIG +RUN printf %s "$ROOTFS_CONFIG" | base64 -d > /config.json && image-env /config.json > /image.env + +FROM ${ROOTFS} AS root +COPY --from=start /image.env /etc/agentenv/vm/image.env + +FROM ${ESP} AS esp + +FROM --platform=$BUILDPLATFORM ${TOOLS} AS disk +ARG TARGETARCH +ARG SIZE_GB +RUN --mount=type=bind,from=root,target=/rootfs --mount=type=bind,from=esp,target=/esp \ + disk "$TARGETARCH" /rootfs /esp /out/disk.img "$SIZE_GB" + +FROM scratch +COPY --from=disk --chown=107:107 /out/disk.img /disk/disk.img diff --git a/src/agent_env/computers/disk/linux/boot/init b/src/agent_env/computers/disk/linux/boot/init new file mode 100755 index 000000000..ba43f8f97 --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/init @@ -0,0 +1,68 @@ +#!/bin/busybox sh +# The disk form's first process: mounts the root filesystem the computer's image became, brings up the network, reads +# the image's start (/etc/agentenv/vm/image.env, from its OCI config) and the deploy's environment (the NoCloud seed's +# user-data), then becomes that start, as its user, as the container runtime would: PID 1 is the image's entrypoint. +/bin/busybox --install -s /bin +export PATH=/bin +mount -t proc proc /proc +mount -t sysfs sysfs /sys +mount -t devtmpfs devtmpfs /dev +log() { echo "agentenv-init: $*" > /dev/console; } +fail() { log "$*"; sleep 2; poweroff -f; } +# decode VALUE: VALUE's base64 decoded into $decoded, its trailing newlines kept, which $(...) alone would drop. +decode() { decoded=$(echo "$1" | base64 -d; echo .); decoded=${decoded%.}; } +while read -r module; do insmod "$module" 2>/dev/null; done < /modules.order +root="" +for _ in $(seq 200); do root=$(findfs LABEL=agentenv-root 2>/dev/null) && break; sleep 0.05; done +[ -n "$root" ] || fail "no filesystem labelled agentenv-root" +mount -t ext4 -o rw,noatime "$root" /newroot || fail "can't mount $root" +ifconfig lo 127.0.0.1 up +iface=$(ls /sys/class/net | grep -vx lo | head -n 1) +[ -n "$iface" ] && ifconfig "$iface" up && timeout 30 udhcpc -i "$iface" -s /udhcpc.script -q -n -t 10 -T 2 -f >/dev/null 2>&1 \ + || log "no DHCP lease on ${iface:-any interface}" +seed=$(findfs LABEL=CIDATA 2>/dev/null || true) +[ -n "$seed" ] && mount -o ro "$seed" /seed +config=/newroot/etc/agentenv/vm/image.env +[ -f "$config" ] || fail "the root filesystem has no $config" +for name in $(env | cut -d= -f1); do case "$name" in PATH|PWD|SHLVL) ;; *) unset "$name" ;; esac; done +user=root workdir=/ +set -- +while read -r line; do + key=${line%%=*} value=${line#*=} + case "$key" in + env) decode "$value"; export "$decoded" ;; + argv) decode "$value"; set -- "$@" "$decoded" ;; + user) decode "$value"; user=$decoded ;; + workdir) decode "$value"; workdir=$decoded ;; + esac +done < "$config" +if [ -f /seed/user-data ]; then + while read -r line; do + key=${line%%=*} value=${line#*=} + case "$key" in ''|\#*) continue ;; esac + decode "$value" + export "$key=$decoded" + done < /seed/user-data +fi +hostname=$( [ -f /seed/meta-data ] && sed -n 's/^local-hostname: *//p' /seed/meta-data) +hostname "${hostname:-computer}" +hostname > /newroot/etc/hostname +printf '127.0.0.1\tlocalhost\n::1\tlocalhost ip6-localhost ip6-loopback\n127.0.1.1\t%s\n' "$(hostname)" > /newroot/etc/hosts +[ -s /resolv.conf ] && cp /resolv.conf /newroot/etc/resolv.conf +entry=$(awk -F: -v u="${user%%:*}" '$1 == u || $3 == u' /newroot/etc/passwd | head -n 1) +[ -n "$entry" ] || fail "the image's user ${user%%:*} isn't in /etc/passwd" +uid=$(echo "$entry" | cut -d: -f3) gid=$(echo "$entry" | cut -d: -f4) +export HOME="${HOME:-$(echo "$entry" | cut -d: -f6)}" HOSTNAME="$(hostname)" +[ "$PATH" = /bin ] && export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +umount /seed 2>/dev/null +[ "$#" -gt 0 ] || fail "the image names no command" +mount --move /sys /newroot/sys +mount --move /dev /newroot/dev +mkdir -p /newroot/dev/pts /newroot/dev/shm +mount -t devpts -o gid=5,mode=620,ptmxmode=666 devpts /newroot/dev/pts +mount -t tmpfs -o mode=1777,size=512m shm /newroot/dev/shm +log "starting $* as $user ($uid:$gid) in $workdir" +# Last, as busybox runs its applets through /proc/self/exe. +mount --move /proc /newroot/proc +exec /bin/busybox switch_root /newroot /usr/bin/setpriv --reuid="$uid" --regid="$gid" --init-groups \ + /usr/bin/env -C "$workdir" "$@" diff --git a/src/agent_env/computers/disk/linux/boot/udhcpc.script b/src/agent_env/computers/disk/linux/boot/udhcpc.script new file mode 100755 index 000000000..13f4bf682 --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/udhcpc.script @@ -0,0 +1,11 @@ +#!/bin/busybox sh +case "$1" in + bound|renew) + ifconfig "$interface" "$ip" netmask "${subnet:-255.255.255.0}" up + for router in ${router:-}; do route add default gw "$router" dev "$interface" && break; done + : > /resolv.conf + for server in ${dns:-}; do echo "nameserver $server" >> /resolv.conf; done + [ -n "${domain:-}" ] && echo "search $domain" >> /resolv.conf + ;; +esac +exit 0 diff --git a/src/agent_env/computers/disk/linux/boot/uki.sh b/src/agent_env/computers/disk/linux/boot/uki.sh new file mode 100755 index 000000000..c7451cd2b --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/uki.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# uki.sh OUT: the one file the disk form's EFI system partition holds, written to OUT under the name UEFI firmware boots +# from removable media: a UKI of Ubuntu's kernel and an initramfs of busybox, init, and the modules a virtio guest needs +# that the kernel doesn't build in, decompressed, since busybox's insmod reads no zstd. +set -eu +out=$1 +kver=$(ls /lib/modules | sort -V | tail -1) +case "$(dpkg --print-architecture)" in + amd64) efi=BOOTX64.EFI stub=linuxx64.efi.stub console=ttyS0 ;; + arm64) efi=BOOTAA64.EFI stub=linuxaa64.efi.stub console=ttyAMA0 ;; +esac +root=/tmp/initramfs +mkdir -p "$root/bin" "$root/dev" "$root/proc" "$root/sys" "$root/newroot" "$root/seed" "$root/lib/modules" "$out" +cp /bin/busybox "$root/bin/busybox" +cp /build/init /build/udhcpc.script "$root/" +chmod 755 "$root/init" "$root/udhcpc.script" +: > "$root/modules.order" +for module in virtio_pci virtio_mmio virtio_blk virtio_scsi virtio_net virtio_rng virtio_console ext4 vfat \ + nls_cp437 nls_iso8859-1 nls_utf8 af_packet; do + modprobe --show-depends -S "$kver" "$module" 2>/dev/null | while read -r kind path _; do + [ "$kind" = insmod ] || continue + name=$(basename "$path" .zst) + grep -qx "/lib/modules/$name" "$root/modules.order" && continue + case "$path" in + *.zst) zstd -qdc "$path" > "$root/lib/modules/$name" ;; + *) cp "$path" "$root/lib/modules/$name" ;; + esac + echo "/lib/modules/$name" >> "$root/modules.order" + done +done +(cd "$root" && find . | sort | cpio -o -H newc --quiet | gzip -9n) > /tmp/initrd.img +# arm64's kernel is a gzipped Image, which the stub can't start; the Image itself is a PE the firmware can. +if [ "$(head -c2 "/boot/vmlinuz-$kver" | od -An -tx1 | tr -d ' ')" = 1f8b ]; then + zcat "/boot/vmlinuz-$kver" > /tmp/vmlinuz +else + cp "/boot/vmlinuz-$kver" /tmp/vmlinuz +fi +ukify build --uname "$kver" --linux /tmp/vmlinuz --initrd /tmp/initrd.img \ + --cmdline "console=$console panic=1 quiet loglevel=4" --stub "/usr/lib/systemd/boot/efi/$stub" --output "$out/$efi" diff --git a/src/agent_env/computers/disk/linux/launcher/Dockerfile b/src/agent_env/computers/disk/linux/launcher/Dockerfile new file mode 100644 index 000000000..9d4f5a1f5 --- /dev/null +++ b/src/agent_env/computers/disk/linux/launcher/Dockerfile @@ -0,0 +1,9 @@ +# The QEMU launcher: boots a Linux computer's disk form, mounted or copied at /disk/disk.img, with its settings in a +# NoCloud seed (see launch). It boots either arch's disk, with KVM where /dev/kvm is passed in and the arch is its own. +FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + qemu-system-x86 qemu-system-arm qemu-utils ovmf qemu-efi-aarch64 dosfstools mtools \ + && rm -rf /var/lib/apt/lists/* +COPY launch seed /usr/local/bin/ +ENTRYPOINT ["/usr/local/bin/launch"] diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch new file mode 100755 index 000000000..e28505419 --- /dev/null +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -0,0 +1,46 @@ +#!/bin/sh +# launch: boot /disk/disk.img, a Linux computer's disk form, under QEMU until it powers off, on a copy-on-write overlay +# so the disk stays as it is: with KVM when /dev/kvm is here and the disk is this machine's arch, else with TCG, from +# UEFI firmware. The guest's console is this container's output. +# VM_ENV_ the computer's variable NAME, its value base64'd, for its NoCloud seed (CIDATA; see seed) +# VM_PORTS guest TCP ports, each forwarded from the same port here +# VM_ARCH the disk's arch, amd64 or arm64; this machine's by default +# VM_CPUS, VM_MEMORY_MB its size: 2 and 4096 by default +set -eu +here=$(dpkg --print-architecture) +arch=${VM_ARCH:-$here} +work=/run/vm +rm -rf "$work" +mkdir -p "$work" +qemu-img create -q -f qcow2 -b /disk/disk.img -F qcow2 "$work/disk.qcow2" +seed "$work" +mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null +mcopy -i "$work/seed.img" "$work/user-data" "$work/meta-data" ::/ +forwards="" +for port in ${VM_PORTS:-}; do + forwards="$forwards,hostfwd=tcp::$port-:$port" +done +accel=tcg cpu=max +if [ -w /dev/kvm ] && [ "$arch" = "$here" ]; then + accel=kvm cpu=host +fi +case "$arch" in + amd64) + cp /usr/share/OVMF/OVMF_VARS_4M.fd "$work/vars.fd" + set -- qemu-system-x86_64 -machine "q35,accel=$accel" \ + -drive if=pflash,format=raw,readonly=on,file=/usr/share/OVMF/OVMF_CODE_4M.fd ;; + arm64) + cp /usr/share/AAVMF/AAVMF_CODE.fd "$work/code.fd" + truncate -s 64M "$work/code.fd" "$work/vars.fd" + set -- qemu-system-aarch64 -machine "virt,accel=$accel" \ + -drive if=pflash,format=raw,readonly=on,file="$work/code.fd" ;; + *) + echo "launch: no architecture $arch" >&2 + exit 2 ;; +esac +exec "$@" -cpu "$cpu" -smp "${VM_CPUS:-2}" -m "${VM_MEMORY_MB:-4096}" \ + -drive if=pflash,format=raw,file="$work/vars.fd" \ + -drive file="$work/disk.qcow2",format=qcow2,if=virtio \ + -drive file="$work/seed.img",format=raw,if=virtio,readonly=on \ + -netdev "user,id=net$forwards" -device virtio-net-pci,netdev=net -device virtio-rng-pci \ + -display none -serial stdio -no-reboot diff --git a/src/agent_env/computers/disk/linux/launcher/seed b/src/agent_env/computers/disk/linux/launcher/seed new file mode 100755 index 000000000..2de80326b --- /dev/null +++ b/src/agent_env/computers/disk/linux/launcher/seed @@ -0,0 +1,10 @@ +#!/bin/sh +# seed DIR: the NoCloud seed's files for the computer launch boots, in DIR: user-data holding each VM_ENV_ variable +# as NAME and its value, base64'd as the variable already holds it, a line each, as the disk's init reads them, and +# meta-data naming the instance. A base64 value has no newline, so no line env prints can pass for another variable. +set -eu +{ + echo "#agentenv-environment" + env | sed -n 's/^VM_ENV_\([A-Za-z_][A-Za-z0-9_]*\)=\([A-Za-z0-9+/=]*\)$/\1=\2/p' | sort +} > "$1/user-data" +printf 'instance-id: %s\nlocal-hostname: computer\n' "$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" > "$1/meta-data" diff --git a/src/agent_env/computers/disk/linux/tools/Dockerfile b/src/agent_env/computers/disk/linux/tools/Dockerfile new file mode 100644 index 000000000..ca10b1e3e --- /dev/null +++ b/src/agent_env/computers/disk/linux/tools/Dockerfile @@ -0,0 +1,7 @@ +# The disk tools: what makes a Linux computer's disk form from a root filesystem without mounting or booting anything, +# so a build runs them unprivileged on its own platform, for either arch. +FROM public.ecr.aws/docker/library/ubuntu:24.04@sha256:24737cef17f4e22b1977df7d9a05e77c4e28b7da5b8911f8d8ccda136bf3573c +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends e2fsprogs dosfstools mtools fdisk qemu-utils python3 \ + && rm -rf /var/lib/apt/lists/* +COPY disk image-env /usr/local/bin/ diff --git a/src/agent_env/computers/disk/linux/tools/disk b/src/agent_env/computers/disk/linux/tools/disk new file mode 100755 index 000000000..7616327cc --- /dev/null +++ b/src/agent_env/computers/disk/linux/tools/disk @@ -0,0 +1,52 @@ +#!/bin/sh +# disk ARCH ROOTFS ESP OUT SIZE_GB: a Linux computer's disk form, written to OUT as qcow2: a GPT disk of an EFI system +# partition and an ext4 root filesystem made from the folder ROOTFS, ownership kept. ESP is a folder holding either a +# disk form's image, disk/disk.img, whose EFI system partition and size the disk keeps byte for byte, or a UKI, which a +# new EFI system partition holds on a disk of SIZE_GB. Nothing is mounted or booted, so it runs unprivileged, on any +# architecture, for either. +set -eu +arch=$1 rootfs=$2 esp=$3 out=$4 size_gb=$5 +case "$arch" in + amd64) uki=BOOTX64.EFI roottype=4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709 ;; + arm64) uki=BOOTAA64.EFI roottype=B921B045-1DF0-41C3-AF44-4C6F280D3FAE ;; + *) echo "disk: no architecture $arch" >&2; exit 2 ;; +esac +esptype=C12A7328-F81F-11D2-BA4B-00A0C93EC93B +sector=512 # bytes a disk sector holds, as sfdisk counts them +mib=$((1024 * 1024 / sector)) # sectors in a MiB, the alignment partitions start on +esp_mib=64 # a new EFI system partition's size +backup_gpt=$mib # sectors kept free at the disk's end, for the backup GPT +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +mkdir -p "$(dirname "$out")" +if [ -f "$esp/disk/disk.img" ]; then + qemu-img convert -O raw "$esp/disk/disk.img" "$work/base.raw" + size=$(stat -c %s "$work/base.raw") + partition=$(sfdisk -d "$work/base.raw" | grep -i "type=$esptype") + esp_start=$(echo "$partition" | sed -n 's/.*start= *\([0-9]*\).*/\1/p') + esp_sectors=$(echo "$partition" | sed -n 's/.*size= *\([0-9]*\).*/\1/p') + [ -n "$esp_start" ] && [ -n "$esp_sectors" ] || { echo "disk: $esp/disk/disk.img has no EFI system partition" >&2; exit 1; } + dd if="$work/base.raw" of="$work/esp.img" bs=$sector skip="$esp_start" count="$esp_sectors" status=none + rm "$work/base.raw" +elif [ -f "$esp/$uki" ]; then + [ -n "$size_gb" ] || { echo "disk: a new disk needs its size, SIZE_GB" >&2; exit 2; } + size=$((size_gb * 1024 * mib * sector)) esp_start=$mib esp_sectors=$((esp_mib * mib)) + mkfs.vfat -C -n ESP "$work/esp.img" $((esp_sectors * sector / 1024)) >/dev/null + mmd -i "$work/esp.img" ::/EFI ::/EFI/BOOT + mcopy -i "$work/esp.img" "$esp/$uki" "::/EFI/BOOT/$uki" +else + echo "disk: $esp holds neither disk/disk.img nor $uki" >&2 + exit 1 +fi +root_start=$(((esp_start + esp_sectors + mib - 1) / mib * mib)) +root_sectors=$((size / sector - root_start - backup_gpt)) +mkfs.ext4 -q -F -L agentenv-root -E root_owner=0:0 -d "$rootfs" "$work/root.img" "$((root_sectors * sector / 1024))k" +truncate -s "$size" "$work/disk.raw" +sfdisk -q "$work/disk.raw" < list[str]: + return [*(f"env={_encoded(variable)}" for variable in config.get("Env") or ()), + *(f"argv={_encoded(arg)}" for arg in [*(config.get("Entrypoint") or ()), *(config.get("Cmd") or ())]), + f"user={_encoded(config.get('User') or 'root')}", f"workdir={_encoded(config.get('WorkingDir') or '/')}"] + + +def _encoded(value: str) -> str: + return base64.b64encode(value.encode()).decode() + + +if __name__ == "__main__": + with open(sys.argv[1]) as file: + print("\n".join(lines(json.load(file)))) diff --git a/src/agent_env/env/bootstrap.py b/src/agent_env/env/bootstrap.py index 1e34ba927..b041dee91 100644 --- a/src/agent_env/env/bootstrap.py +++ b/src/agent_env/env/bootstrap.py @@ -23,7 +23,14 @@ from agent_env.env.envs import GatewayEnv, MCPServerEnv from agent_env.env.envs.service_db import ServiceDBEnv from agent_env.env.envs.playwright_mcp import PLAYWRIGHT_MCP_ENVIRONMENT_NAME, PLAYWRIGHT_MCP_IMAGE_TAG -from agent_env.env.published_images import BUNDLE_IMAGES, PublishedImage, published_digest +from agent_env.env.published_images import ( + BUNDLE_IMAGES, + DISK_IMAGES, + DISK_TOOLS_IMAGE, + LAUNCHER_IMAGE, + PublishedImage, + published_digest, +) from agent_env.store.base import NotFoundError from agent_env.store.document_store import LocalSqliteDocumentStore from agent_env.store.ids import derive_id @@ -155,14 +162,18 @@ def _metadata(kind: str, extra: Mapping[str, Any] | None) -> dict[str, Any]: SERVICE_DB: [SERVICE_DB_DOCKERFILE, DB_WEB_DOCKERFILE, DB_MCP_DOCKERFILE], WEBSITE_BROWSER: [PLAYWRIGHT_MCP_DOCKERFILE], } -# The image each shipped Dockerfile is published as: the infra envs', the shipped agents' and an example env's. +# The image each shipped Dockerfile is published as: the infra envs', the disk tools and QEMU launcher, the shipped +# agents', an example env's, the base computers' and their disk forms. PUBLISHED_IMAGES = ( PublishedImage("agentenv-gateway", GATEWAY_DOCKERFILE), PublishedImage("agentenv-service-db", SERVICE_DB_DOCKERFILE), PublishedImage("agentenv-db-web", DB_WEB_DOCKERFILE), PublishedImage("agentenv-db-mcp", DB_MCP_DOCKERFILE), PublishedImage("agentenv-playwright-mcp", PLAYWRIGHT_MCP_DOCKERFILE), + DISK_TOOLS_IMAGE, + LAUNCHER_IMAGE, *BUNDLE_IMAGES, + *DISK_IMAGES, ) _PUBLISHED = {image.dockerfile: image for image in PUBLISHED_IMAGES} diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 279a5ad73..a82808e90 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -23,7 +23,7 @@ deploy_through_provider, provider_or_class, ) -from agent_env.env.envs.computer.utils import composer, gate, inside +from agent_env.env.envs.computer.utils import boot, composer, gate, inside from agent_env.env.envs.computer.utils.composer import Composing from agent_env.env.envs.computer.utils import params as computer_params from agent_env.env.envs.computer.driver_env import ComputerDriverEnv @@ -45,6 +45,8 @@ DEFAULT_DESTINATION = "Downloads" #: The computer app env's type: an app composed into a computer env, and a computer env of its own. APP_TYPE = "computer_app" +#: The statuses of a build a deploy boots, by form: a disk may also boot unprobed, as KVM is where a disk is probed. +_DEPLOYABLE = {"container": ("ready",), "disk": ("ready", "unverified")} class ComputerEnv(Env): @@ -160,16 +162,29 @@ def app_defaults(self) -> dict[str, dict[str, Any]]: return {app.environment_name: {**app.settings, **self.app_settings.get(app.environment_name, {})} for app in self.composed_apps()} + def computer_values(self) -> dict[str, Any]: + """The env's values of its base's settings: its ``computer_settings``, and its display as the resolution.""" + display = self.display + return {**self.computer_settings, **({"resolution": f"{display['width']}x{display['height']}"} if display else {})} + def computer_artifact(self) -> ComputerArtifact: """The env's computer: its base composed with its driver, over the images built.""" return ComputerArtifact.get(self.metadata["computer"]["id"], self.metadata["computer"]["version"]) - def ready_builds(self, form: Optional[str] = None) -> list[tuple[str, str, dict]]: - """Each ready build, as (form, arch, record), in the base's order of forms and of arches; only - ``form``'s when it's given.""" + def deployable_builds(self, form: Optional[str] = None) -> list[tuple[str, str, dict]]: + """Each build a deploy may boot, as (form, arch, record), in the base's order of forms and of arches; only + ``form``'s when it's given: a ready one, or a disk no compose could probe (unverified), which its first boot on + a provider that boots it natively probes before it's used.""" composed, arches = self.metadata.get("composed", {}), self.computer.arch return [(name, arch, composed[name][arch]) for name in gate.forms(self.computer) if form in (None, name) - for arch in arches if composed.get(name, {}).get(arch, {}).get("status") == "ready"] + for arch in arches if composed.get(name, {}).get(arch, {}).get("status") in _DEPLOYABLE[name]] + + def disk_size_refusal(self, form: str, disk_size_gb: float) -> Optional[str]: + """Why a deploy of ``form`` can't have ``disk_size_gb``: a disk is the size its base makes it, which a deploy + doesn't enlarge.""" + if form == "disk" and disk_size_gb > (size := self.computer.forms.disk.min_disk_gb): + return f"{self.id}'s disk is {size} GB, which a deploy doesn't enlarge, and it asks for {disk_size_gb:g} GB" + return None def build_statuses(self) -> dict[str, str]: return {f"{form}/{arch}": result.get("status") for form, by_arch in self.metadata.get("composed", {}).items() @@ -196,12 +211,13 @@ def deploy_refusal(self, *, sandbox_type: str | None = None, env_params: Optiona if refusal := deploy_refusal(self, provider_or_class(self), {**options, "env_params": env_params}): return refusal - _, _, problems = computer_params.checked(self, env_params) + params, _, problems = computer_params.checked(self, env_params) sandbox_provider = build_sandbox_provider(sandbox_type) if sandbox_type else get_env_sandbox_provider() links, arches = sandbox_provider.links, gate.arches(self.computer, self.driver, self.composed_apps()) - if not any(f"linux/{arch}" in link.native_platforms() for link in links for arch in arches): + forms = [params.form] if params.form else gate.forms(self.computer) + if not any(boot.native_links(sandbox_provider, arch, form) for form in forms for arch in arches): problems.append(f"{', '.join(type(link).__name__ for link in links)} runs none of its arches " - f"({', '.join(arches)}) natively") + f"({', '.join(arches)}) natively as {' or '.join(forms)}") policy = NetworkPolicy.from_dict(self.network) if self.network else None if policy is not None and not any(link.supports_network_policy(policy) for link in links): problems.append(f"{', '.join(type(link).__name__ for link in links)} can't enforce its network") diff --git a/src/agent_env/env/envs/computer/utils/boot.py b/src/agent_env/env/envs/computer/utils/boot.py index c03585b51..cb4081c0d 100644 --- a/src/agent_env/env/envs/computer/utils/boot.py +++ b/src/agent_env/env/envs/computer/utils/boot.py @@ -1,6 +1,6 @@ -"""Booting a composed computer: one build of it in a container, on a sandbox provider that runs its arch natively, with -its settings in the environment the base and the driver read them from, until its driver says it's ready. A deploy and -gate 3 boot computers this way.""" +"""Booting a composed computer: one build of it, its container in a container or its disk in a VM, on a sandbox provider +that runs its arch natively, with its settings in the environment the base and the driver read them from (a disk's in +its NoCloud seed), until its driver says it's ready. A deploy and gate 3 boot computers this way.""" from __future__ import annotations @@ -38,6 +38,8 @@ WITHHELD = "" #: How many random bytes a computer's control token carries: 256 bits, past guessing. CONTROL_TOKEN_BYTES = 32 +#: The size of a disk's VM when a deploy sets none: what the desktop was measured booting and passing the probe in. +DISK_CPU, DISK_MEMORY_MB = 2.0, 4096 @dataclass @@ -47,9 +49,11 @@ class Booted: card: dict -def native_links(sandbox_provider: SandboxProvider, arch: str) -> list[SandboxProvider]: - """The providers a boot of ``arch`` may run on, in the order a chain tries them: those that run it natively.""" - return [link for link in sandbox_provider.links if f"linux/{arch}" in link.native_platforms()] +def native_links(sandbox_provider: SandboxProvider, arch: str, form: str = "container") -> list[SandboxProvider]: + """The providers a boot of ``form`` for ``arch`` may run on, in the order a chain tries them: those that run its + container, or boot its disk, natively.""" + return [link for link in sandbox_provider.links if f"linux/{arch}" in ( + link.disk_image_platforms() if form == "disk" else link.native_platforms())] def settings_environment(computer: ComputerArtifact, values: dict[str, Any], *, environment_name: str, @@ -114,13 +118,20 @@ def control_environment(token: Optional[str]) -> dict[str, str]: async def boot(sandbox_provider: SandboxProvider, *, image: str, port: int, environment: dict[str, str], token: str, network_policy: Optional[NetworkPolicy], timeout: int, attribution: Optional[Attribution], disk_size_gb: float = 10, cpu: Optional[float] = None, memory_mb: Optional[int] = None, - ready_s: float = READY_TIMEOUT_S) -> Booted: - """``image`` booted with ``environment`` and its control ``token`` on ``sandbox_provider``, which the caller chose - to run its arch natively, its driver's ``port`` reached on the sandbox, once the driver, asked with that token, says - the computer is ready; a boot that fails is torn down.""" - sandbox = await sandbox_provider.create_container( - image_name=image, port=port, env={**environment, **control_environment(token)}, network_policy=network_policy, - disk_size_gb=disk_size_gb, timeout=timeout, attribution=dict(attribution or {}), **_size_kwargs(cpu, memory_mb)) + ready_s: float = READY_TIMEOUT_S, form: str = "container") -> Booted: + """``image``, of ``form``, booted with ``environment`` and its control ``token`` on ``sandbox_provider``, which the + caller chose to run its arch natively, its driver's ``port`` reached on the sandbox, once the driver, asked with that + token, says the computer is ready; a boot that fails is torn down.""" + environment = {**environment, **control_environment(token)} + if form == "disk": + sandbox = await sandbox_provider.create_vm( + image=image, env=environment, exposed_ports=[port], network_policy=network_policy, + disk_size_gb=disk_size_gb, timeout=timeout, attribution=dict(attribution or {}), + cpu=cpu or DISK_CPU, memory=memory_mb or DISK_MEMORY_MB) + else: + sandbox = await sandbox_provider.create_container( + image_name=image, port=port, env=environment, network_policy=network_policy, disk_size_gb=disk_size_gb, + timeout=timeout, attribution=dict(attribution or {}), **_size_kwargs(cpu, memory_mb)) try: url = sandbox.tunnel_urls[port] return Booted(sandbox, url, await ready(url, ready_s, token)) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 4904407ed..d71c7504c 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -7,15 +7,18 @@ import asyncio import contextlib +import functools import hashlib import io import json +import re import logging import os import shlex import shutil import subprocess import tarfile +import time import uuid from abc import ABC, abstractmethod from collections.abc import Iterator @@ -28,6 +31,7 @@ from agent_env.artifact.artifacts.docker_image import docker_login, push_local_image from agent_env.config import get_config from agent_env.env.envs.computer.utils import layers_in_image +from agent_env.env.published_images import COMPUTER_IMAGES from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM @@ -125,9 +129,18 @@ def mark_runnable(self, paths: list[str]) -> None: """Make each of ``paths`` executable that starts as a script or a Linux binary does.""" @abstractmethod - def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: - """Build ``context`` with ``dockerfile``, written into it, as the image ``tag`` for ``platform``. Raises - DockerBuildError.""" + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT, + build_args: dict[str, str] | None = None) -> None: + """Build ``context`` with ``dockerfile``, written into it, and ``build_args``, as the image ``tag`` for + ``platform``. Raises DockerBuildError.""" + + @abstractmethod + def image_config(self, tag: str) -> dict: + """The ``config`` object of the image config of ``tag``, an image its Docker holds.""" + + @abstractmethod + def login(self, auth) -> None: + """Sign its Docker in to the registry ``auth``, image store credentials or None, names.""" @abstractmethod def installed(self, tag: str, prefix: str) -> dict[str, str]: @@ -156,9 +169,9 @@ def __init__(self, provider: SandboxProvider, store_id: str, repository: str, em self.provider, self.emulate = provider, emulate self._refused = [(link, _refused(link, store_id, repository)) for link in provider.links] - def link(self, platform: str) -> SandboxProvider: - """The link that builds ``platform``. Raises Unbuildable, or Skipped when only emulation would build it and the - put turned emulation off.""" + def link(self, platform: str, base: DockerImageArtifact) -> SandboxProvider: + """The link that builds ``platform`` ``FROM`` ``base``. Raises Unbuildable, or Skipped when only emulation would + build it and the put turned emulation off.""" links = [link for link, problem in self._refused if problem is None] if native := [link for link in links if platform in link.native_platforms()]: return native[0] @@ -169,7 +182,7 @@ def link(self, platform: str) -> SandboxProvider: raise Skipped("emulation is off, and " + "; ".join( f"{type(link).__name__} doesn't run {platform} natively" for link, _ in self._refused)) for link in links: - if not _docker_here(link) or platform in _emulated_platforms(): + if not _docker_here(link) or _runs_here(platform, _platform_check(base)): return link unemulated = f"this machine can't build {platform}: its Docker has no emulation for it" raise Unbuildable("; ".join(problem or unemulated for _, problem in self._refused)) @@ -227,14 +240,30 @@ def _docker_here(link: SandboxProvider) -> bool: return link.ON_THIS_MACHINE and link.REACHES_HOST_AT is None -def _emulated_platforms() -> set[str]: - """The platforms this machine's Docker builds, its own and those it emulates.""" - inspected = subprocess.run(["docker", "buildx", "inspect", "--bootstrap"], capture_output=True, text=True) - platforms = set() - for line in inspected.stdout.splitlines(): - if line.strip().startswith("Platforms:"): - platforms |= {item.strip().rstrip("*") for item in line.split(":", 1)[1].split(",")} - return platforms +@functools.cache +def _runs_here(platform: str, image: str) -> bool: + """Whether this machine's Docker runs ``platform``'s programs, natively or emulated, as a container of ``image`` for + it shows: what its builder lists can leave out a platform it runs, as colima's does Rosetta's linux/amd64.""" + try: + return subprocess.run(["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", image], + capture_output=True).returncode == 0 + except FileNotFoundError: + return False + + +def _platform_check(base: DockerImageArtifact) -> str: + """The image ``_runs_here`` starts for a build ``FROM`` ``base``, one the build pulls anyway: ``base`` when it's a + registry's, signed in to it, else the image the shipped Linux base builds from, as its build pulls it.""" + if not names_registry(base.image_name): + return _shipped_base_from() + docker_login(get_config().get_image_store().auth(base.pinned_name)) + return base.pinned_name + + +@functools.cache +def _shipped_base_from() -> str: + """The image the shipped Linux base computer's Dockerfile builds ``FROM``, pinned by its digest there.""" + return re.search(r"^FROM\s+(\S+)", COMPUTER_IMAGES[0].dockerfile.read_text(), re.MULTILINE).group(1) def _resolver(source: str, arch: str, context: str, out: str) -> list[str]: @@ -253,17 +282,28 @@ def __init__(self, name: str, work: Path, base: DockerImageArtifact, platform: s build_for(base, platform) def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: + """With the build context and what the resolver reads copied into its container, and what it wrote copied out, + rather than bind-mounted: a Docker in a VM, as on a Mac, mounts only the folders the VM shares, colima's only the + home, and keeps no owner on them.""" out = self.work / RESOLVED / "out" out.mkdir(parents=True) (out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]})) shutil.copy(RESOLVER, out / "resolve.py") + container, deadline = f"agentenv-resolve-{uuid.uuid4().hex[:12]}", time.monotonic() + timeout try: - ran = subprocess.run(_resolver(source, arch, str(self.work / CONTEXT), str(out)), capture_output=True, - text=True, timeout=timeout) + _docker("create", "--name", container, "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", + "python3", source, "/out/resolve.py", "/out/request.json", "/out", deadline=deadline) + _docker("cp", f"{self.work / CONTEXT}/.", f"{container}:/ctx", deadline=deadline) + _docker("cp", f"{out}/.", f"{container}:/out", deadline=deadline) + ran = subprocess.run(["docker", "start", "--attach", container], capture_output=True, text=True, + timeout=max(deadline - time.monotonic(), 0)) + if ran.returncode != 0: + raise ResolveError("resolving the packages failed", ran.stdout + ran.stderr) + _docker("cp", f"{container}:/out/.", str(out), deadline=deadline) except subprocess.TimeoutExpired as e: raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None - if ran.returncode != 0: - raise ResolveError("resolving the packages failed", ran.stdout + ran.stderr) + finally: + subprocess.run(["docker", "rm", "--force", container], capture_output=True) return json.loads((out / "result.json").read_text()) def download(self, url: str, sha256: str, path: str) -> None: @@ -310,10 +350,21 @@ def mark_runnable(self, paths: list[str]) -> None: if file.read(4).startswith(RUN_ONLY): (self.work / path).chmod(EXECUTABLE) - def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT, + build_args: dict[str, str] | None = None) -> None: (self.work / context).mkdir(parents=True, exist_ok=True) (self.work / context / "Dockerfile").write_text(dockerfile) - build_image(self.work / context / "Dockerfile", self.work / context, tag, platform=platform, labels=labels) + build_image(self.work / context / "Dockerfile", self.work / context, tag, platform=platform, labels=labels, + build_args=build_args) + + def image_config(self, tag: str) -> dict: + inspected = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", tag], + capture_output=True, text=True, check=True) + return json.loads(inspected.stdout) + + def login(self, auth) -> None: + if auth is not None: + docker_login(auth) def installed(self, tag: str, prefix: str) -> dict[str, str]: created = subprocess.run(["docker", "create", tag], capture_output=True, text=True, check=True).stdout.strip() @@ -423,15 +474,23 @@ def place(self, copies: dict[str, str], fetches: dict[str, tuple[str, str]]) -> def mark_runnable(self, paths: list[str]) -> None: self._each([f"{self.helper} runnable {self._at(path)}" for path in paths]) - def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT, + build_args: dict[str, str] | None = None) -> None: folder = f"{self.work}/{context}" self._write(f"{folder}/Dockerfile", dockerfile.encode()) - options = " ".join(f"--label {shlex.quote(f'{name}={value}')}" for name, value in labels.items()) + options = " ".join([*(f"--label {shlex.quote(f'{name}={value}')}" for name, value in labels.items()), + *(f"--build-arg {shlex.quote(f'{name}={value}')}" for name, value in (build_args or {}).items())]) code, stdout, stderr = self._output(f"cd {folder} && docker build --platform {shlex.quote(platform)} {options} " f"-f Dockerfile -t {shlex.quote(tag)} . 2>&1") if code != 0: raise DockerBuildError(f"docker build of {tag} failed (exit {code}):\n{(stdout + stderr).rstrip()}") + def image_config(self, tag: str) -> dict: + return json.loads(self._exec(f"docker image inspect --format '{{{{json .Config}}}}' {shlex.quote(tag)}")) + + def login(self, auth) -> None: + self._login(auth) + def installed(self, tag: str, prefix: str) -> dict[str, str]: listed = self._exec(f"{self.helper} installed {shlex.quote(tag)} {shlex.quote(f'{prefix}/installed.json')}") return _packages(listed.encode()) if listed.strip() else {} @@ -536,6 +595,15 @@ def _packages(listed: bytes) -> dict[str, str]: return {package["name"]: package["version"] for package in json.loads(listed)} +def _docker(*args: str, deadline: float) -> str: + """What ``docker`` printed for ``args``, by the ``time.monotonic()`` ``deadline``; a failure raises, with what it + said, and running past the deadline raises TimeoutExpired.""" + ran = subprocess.run(["docker", *args], capture_output=True, text=True, timeout=max(deadline - time.monotonic(), 0)) + if ran.returncode != 0: + raise RuntimeError(f"docker {args[0]} failed: {ran.stderr.strip()}") + return ran.stdout + + def save_image(tag: str, path: Path) -> None: subprocess.run(["docker", "save", "-o", str(path), tag], check=True, capture_output=True, text=True) diff --git a/src/agent_env/env/envs/computer/utils/composed.py b/src/agent_env/env/envs/computer/utils/composed.py new file mode 100644 index 000000000..4d4d924e7 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/composed.py @@ -0,0 +1,90 @@ +"""A computer env's ``composed`` metadata: each build's result by form and arch, recorded as it finishes.""" + +from __future__ import annotations + +import copy +import threading +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.env.env import Env +from agent_env.store.base import ConcurrentModificationError +from agent_env.store.ids import derive_id + +if TYPE_CHECKING: + from agent_env.env.envs.computer.env import ComputerEnv + +# A pair's statuses once its image is in the store. +BUILT = ("unverified", "ready") +_WRITE_TRIES = 4 + + +class Record: + """The env's ``composed`` metadata, by form and arch. Each result is merged into the stored table pair by pair, the + table read again when another build wrote it first, so builds of one env at once keep each other's pairs, and a + result never undoes one that got further (``_rank``).""" + + def __init__(self, env: ComputerEnv): + self.env, self.lock = env, threading.Lock() + + def started(self, pairs: list[tuple[str, str]]) -> None: + at = now() + self._merge({pair: {"status": "building", "started_at": at} for pair in pairs}) + + def finished(self, form: str, arch: str, result: dict) -> None: + self._merge({(form, arch): result}) + + def built(self, form: str = "container") -> dict[str, str]: + """Each arch's image of ``form``, of the pairs built or reused.""" + return {arch: result["image"] for arch, result in self.env.metadata.get("composed", {}).get(form, {}).items() + if result.get("status") in BUILT} + + def computer(self, ref: dict | None) -> None: + """Record the env's computer, with the images it was written over, the disks' apart.""" + if ref is not None: + with self.lock: + disks = self.built("disk") + self.env.merge_metadata({"computer": {**ref, "images": self.built(), **({"disks": disks} if disks else {})}}) + + def _merge(self, results: dict[tuple[str, str], dict]) -> None: + with self.lock: + for attempt in range(_WRITE_TRIES): + composed = copy.deepcopy(self.env.metadata.get("composed", {})) + for (form, arch), result in results.items(): + stored = composed.setdefault(form, {}).get(arch, {}) + if _rank(result) >= _rank(stored): + composed[form][arch] = {"started_at": stored.get("started_at"), **result} + try: + self.env.update_metadata({**self.env.metadata, "composed": composed}) + return + except ConcurrentModificationError: + if attempt == _WRITE_TRIES - 1: + raise + self.env.metadata = Env.get(self.env.stored_id, self.env.version).metadata + + +def failed(env: ComputerEnv, form: str, arch: str, stage: str, key: str | None, reason: str, + log: str | None = None) -> dict: + result = {"status": "failed", "stage": stage, "reason": reason, "failed_at": now()} + if key is not None: + result["key"] = key + if log is not None: + stored = FileArtifact.put_bytes(derive_id(env.stored_id, f"build_log_{form}_{arch}"), description=( + f"the {form}/{arch} build of {env.stored_id} v{env.version}, which failed"), filename="build.log", + content=log.encode(), content_type="text/plain") + result["log"] = {"id": stored.id, "version": stored.version} + result["reason"] = f"{reason}: {log.strip().splitlines()[-1] if log.strip() else 'no output'}" + return result + + +def now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds") + + +def _rank(result: dict) -> int: + """How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an + image. A result takes a stored one's place only when it got at least as far.""" + if result.get("status") == "ready": + return 2 + return 1 if result.get("status") == "unverified" or result.get("stage") == "probe" else 0 diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index c256c287e..e0c4112c0 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -1,14 +1,16 @@ """The composer: a computer env's base, driver and apps built into one image for each form and arch, on a builder the put's sandbox provider gives (``builder``), tagged in the env's image store by a key of everything that goes into it, so -an image of that key, built for any env and on any builder, is reused. +an image of that key, built for any env and on any builder, is reused. A disk form is made from its arch's container +once that has built: its root filesystem as the disk's, on the base disk's EFI system partition, by a build that runs +only on the builder's own platform. Each result is recorded in the env's metadata as it finishes (``composed``), and the arches that built are written as the env's computer, a ``computer`` artifact over an index of their images.""" from __future__ import annotations import asyncio +import base64 import contextlib -import copy import hashlib import importlib.metadata import json @@ -20,36 +22,32 @@ import threading import time import urllib.parse -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import dataclass -from datetime import datetime, timezone from pathlib import Path, PurePosixPath -from collections.abc import Callable +from collections.abc import Callable, Iterator from typing import TYPE_CHECKING, Any from agentenv_protocol.computer import apps as installed_apps from agentenv_protocol.computer.app import BUILD_SETTINGS, SOCKET -from agentenv_protocol.computer.conformance import run_probe from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact -from agent_env.artifact.artifacts.file import FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.config import get_config from agent_env.entity.export import Unwritable, checked_path -from agent_env.env.env import Env -from agent_env.env.envs.computer.utils import boot, builder, clashes, gate, packages +from agent_env.env.envs.computer.utils import boot, builder, clashes, gate, packages, probing +from agent_env.env.envs.computer.utils.composed import BUILT, Record, failed, now from agent_env.env.envs.computer.utils.install import Install +from agent_env.env.published_images import DISK, DISK_TOOLS_IMAGE, published_disk, published_pinned from agent_env.providers.sandbox_providers.host_images import platform_tag -from agent_env.store.base import ConcurrentModificationError from agent_env.store.ids import derive_id -from agent_env.store.image_store.image_contents import labels +from agent_env.store.image_store.image_contents import image_config, labels from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.registry_api import NoSuchImageError, pin_digest, write_index from agent_env.utils.docker.build import DockerBuildError if TYPE_CHECKING: - from agentenv_protocol.computer.conformance import ProbeReport from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider from agent_env.env.envs.computer.env import ComputerEnv @@ -71,13 +69,6 @@ KEY_LABEL = "dev.agentenv.computer-key" # What a build installed and whether it was emulated, kept on its image for an env that reuses it. INSTALLED_LABEL, EMULATED_LABEL = "dev.agentenv.installed", "dev.agentenv.emulated" -# A pair's statuses once its image is in the store. -_BUILT = ("unverified", "ready") -_WRITE_TRIES = 4 -#: How long a probe's computer may run before its sandbox provider stops it. -_PROBE_TTL_S = 1800 -#: How long the probe at full may take on a computer that booted: about ten seconds on a healthy one. -_PROBE_TIMEOUT_S = 120 #: How often a compose says a pair is still at a stage, which can take most of an hour under emulation. _HEARTBEAT_S = 600 _PROTOCOL = "agentenv-framework-protocol" @@ -110,25 +101,30 @@ def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, compos builds = builder.Builds(provider, env.stored_id, REPOSITORY, composing.emulate) pairs = pairs if pairs is not None else [(form, arch) for form in gate.forms(base) for arch in gate.arches(base, driver, env.composed_apps())] - record = _Record(env) + record = Record(env) record.started(pairs) with _Progress(env, composing.on_progress) as progress, ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: progress.say(f"composing {', '.join(f'{form}/{arch}' for form, arch in pairs)} on " f"{' or '.join(type(link).__name__ for link in provider.links)}") - futures = {pool.submit(_composed, env, base, driver, form, arch, builds, progress): (form, arch) - for form, arch in pairs} + containers = {arch: pool.submit(_composed, env, base, driver, form, arch, builds, progress) + for form, arch in pairs if form == "container"} + futures = {future: ("container", arch) for arch, future in containers.items()} + futures |= {pool.submit(_composed_disk, env, base, driver, arch, containers.get(arch), builds, progress): + ("disk", arch) for form, arch in pairs if form == "disk"} for future in as_completed(futures): form, arch = futures[future] result = future.result() progress.ended(f"{form}/{arch}", result) record.finished(form, arch, result) - record.computer(_computer(env, base, record.built())) + record.computer(_computer(env, base, record.built(), record.built("disk"))) def current(env: ComputerEnv) -> bool: """Whether the env's computer was written over every pair built, as a put or env build that failed writing it leaves it not.""" - return "computer" in env.metadata and env.metadata["computer"].get("images") == _Record(env).built() + computer, record = env.metadata.get("computer"), Record(env) + return computer is not None and computer.get("images") == record.built() and computer.get( + "disks", {}) == record.built("disk") def pending(env: ComputerEnv, sandbox_provider: SandboxProvider | None = None) -> list[tuple[str, str]]: @@ -141,7 +137,7 @@ def pending(env: ComputerEnv, sandbox_provider: SandboxProvider | None = None) - for form in gate.forms(env.computer) for arch in gate.arches(env.computer, env.driver, env.composed_apps())} return [(form, arch) for (form, arch), status in statuses.items() - if status not in _BUILT or (status == "unverified" and boot.native_links(provider, arch))] + if status not in BUILT or (status == "unverified" and boot.native_links(provider, arch, form))] def _composed(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, @@ -153,141 +149,228 @@ def _composed(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEn return _probed(env, driver, form, arch, result, builds.provider, progress) +def _composed_disk(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, arch: str, + container: Future | None, builds: builder.Builds, progress: _Progress) -> dict: + """The disk form of ``arch``, once its container, composed now or before, has built: built or reused, then probed + when it built (gate 3).""" + built = container.result() if container is not None else env.metadata.get("composed", {}).get( + "container", {}).get(arch, {}) + if built.get("status") not in BUILT: + return {"status": "skipped", "reason": f"its container form didn't build ({built.get('status', 'never')})", + "skipped_at": now()} + result = _built_disk(env, base, arch, built, builds, progress) + if result["status"] != "unverified": + return result + return _probed(env, driver, "disk", arch, result, builds.provider, progress) + + def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, result: dict, sandbox_provider: SandboxProvider, progress: _Progress) -> dict: - """Gate 3: the build of ``result`` booted on the first link of ``sandbox_provider`` that runs ``arch`` natively, and - probed at ``full``: ready, with the driver's card, or failed at the probe; each with a summary of the probe and its - report stored. Still unverified, saying why, when no link runs ``arch`` natively.""" - if not (links := boot.native_links(sandbox_provider, arch)): - return {**result, "reason": f"no link of the sandbox provider runs linux/{arch} natively to probe it"} + """Gate 3: the build of ``result`` booted on the first link of ``sandbox_provider`` that runs ``arch``'s ``form`` + natively, and probed at ``full`` (``probing.checked``). Still unverified, saying why, when no link does.""" + if not (links := boot.native_links(sandbox_provider, arch, form)): + how = f"boots linux/{arch} disks" if form == "disk" else f"runs linux/{arch}" + return {**result, "reason": f"no link of the sandbox provider {how} natively to probe it"} progress.at(f"{form}/{arch}", "probing", f"probing on {type(links[0]).__name__}") - try: - report, card, provider = asyncio.run(_probe(links, env, driver, result["image"])) - except TimeoutError: - return _failed(env, form, arch, "probe", result["key"], f"the probe didn't finish within {_PROBE_TIMEOUT_S} s") - except Exception as e: - return _failed(env, form, arch, "probe", result["key"], f"it didn't boot: {e}") - stored = FileArtifact.put_bytes(derive_id(env.stored_id, f"probe_report_{form}_{arch}"), description=( - f"the {form}/{arch} probe of {env.stored_id} v{env.version}"), filename="probe.json", - content=report.model_dump_json(indent=1).encode(), content_type="application/json") - probe = {"level": report.level, "provider": provider, "passed": report.passed, "checks": report.counts(), - "extensions": report.extensions, "report": {"id": stored.id, "version": stored.version}, - "probed_at": _now()} - if not report.passed: - failing = "; ".join(f"{c.check} {c.name}: {c.detail}" for c in report.checks if c.status == "failed") - return {**result, "status": "failed", "stage": "probe", "reason": f"the probe failed: {failing}", - "failed_at": _now(), "probe": probe} - return {**result, "status": "ready", "card": card, "probe": probe} - - -async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, - image: str) -> tuple[ProbeReport, dict, str]: - """``image`` booted at the env's settings and its apps' defaults with a control token of its own, as a deploy with - no ``env_params`` boots it, on the first of ``links`` that boots it, probed at ``full`` with that token within - ``_PROBE_TIMEOUT_S`` and torn down; the report, the driver's card and the provider's name.""" - token = boot.control_token() - environment = {**boot.settings_environment(env.computer, _values(env), environment_name=env.environment_name), - **boot.app_settings_environment(env.composed_apps(), env.app_defaults(), env.computer.os.type)} - problems = [] - for link in links: - try: - booted = await boot.boot(link, image=image, port=driver.port, environment=environment, token=token, - network_policy=None, timeout=_PROBE_TTL_S, attribution=None, - ready_s=boot.ready_timeout(env.composed_apps(), env.computer.os.type)) - except Exception as e: - problems.append(f"{type(link).__name__}: {e}") - continue - try: - async with asyncio.timeout(_PROBE_TIMEOUT_S): - return await run_probe(booted.url, "full", token=token), booted.card, type(link).__name__ - finally: - await boot.discard(booted.sandbox) - raise RuntimeError("; ".join(problems)) + return asyncio.run(probing.checked(links, env, driver, form, arch, result)) def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, builds: builder.Builds, progress: _Progress) -> dict: """The status of ``form`` and ``arch`` once built and pushed, or reused, or the stage that failed and why, each stage reported to ``progress``. A builder from ``builds`` resolves and builds it, opened when first needed.""" - stage, key, pair = "build", None, f"{form}/{arch}" + pair = f"{form}/{arch}" + with (_attempting(env, form, arch) as attempt, tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, + contextlib.ExitStack() as opened): + context, fetched, from_store = Path(work) / builder.CONTEXT, {}, {} + context.mkdir() + source, base_identity = _base(base, arch) + layer = _stage_driver(context, driver, base, from_store) + apps = [_stage_app(context, app, base, from_store) for app in env.composed_apps()] + layers = [layer, *apps] + wants = _wants(layers, _stage_protocol(context)) + platform, held = f"linux/{arch}", [] + + def on() -> builder.Builder: + if not held: + link = builds.link(platform, base.image()) + native = platform in link.native_platforms() + progress.at(pair, "starting its builder", f"building natively {builder.where(link)}" if native else + f"building emulated, which is slow, {builder.where(link)}; " + f"{builder.instead_of_emulating(builds.provider, platform)}") + held.append(opened.enter_context(builds.open(link, platform, Path(work), base.image(), from_store))) + return held[0] + + inputs = packages.inputs_digest(wants, context, base_identity, arch) + lock = packages.stored(inputs, env.stored_id) if inputs is not None else None + if inputs is not None and lock is None: + attempt.stage = "resolve" + resolving = on() + progress.at(pair, "resolving packages") + lock, fetched = packages.resolve(inputs, wants, resolving, source, arch, env.stored_id) + progress.at(pair, "resolved", f"resolved {len(fetched)} files and stored them") + attempt.stage = "build" + dockerfile = _dockerfile(source, layers, base, _build_settings(env, base), lock) + key = attempt.key = _key(base=base_identity, dockerfile=dockerfile, context=_digest(context), + lock=lock.digest() if lock is not None else None, form=form, arch=arch, format=FORMAT) + store, ref, pinned = _stored(env, key) + if pinned is not None: + progress.at(pair, "reusing", "reusing the image of the same key, built earlier") + kept = labels(pinned, arch, store.auth(pinned)) + return {"status": "unverified", "key": key, "image": pinned, "reused": True, + "emulated": kept.get(EMULATED_LABEL) == "true", "lock": inputs, + "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": now()} + building_on = on() + progress.at(pair, "building the image") + if lock is not None: + packages.stage(lock, building_on, env.stored_id, fetched) + local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform + building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) + if reason := _compare(building_on, local, layers, pair, progress): + return failed(env, form, arch, attempt.stage, key, reason) + installed = building_on.installed(local, layer.prefix) + building_on.build(f"FROM {local}\n", local, platform, { + INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", + }, context="label") + attempt.stage = "push" + progress.at(pair, "pushing the image") + building_on.push(local, ref, store) + return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, + "emulated": emulated, "builder": building_on.name, "lock": inputs, "installed": installed, + "built_at": now()} + return attempt.result + + +def _built_disk(env: ComputerEnv, base: ComputerArtifact, arch: str, container: dict, builds: builder.Builds, + progress: _Progress) -> dict: + """The status of the disk form of ``arch`` once built from ``container``, its container's result, and pushed, or + reused, or the stage that failed and why. Its key is of the container's key, the base disk and the disk recipe; + the base disk is the base's own, the one published for it, or else one made on the builder from the base's + container and the boot recipe. Its build runs no program of ``arch``'s but that recipe's.""" + pair, platform = f"disk/{arch}", f"linux/{arch}" + with (_attempting(env, "disk", arch) as attempt, + tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened): + context = Path(work) / builder.CONTEXT + shutil.copytree(DISK / "boot", context) + shutil.copytree(DISK / "tools", context / "tools") + base_disk, base_identity, recipe = *_base_disk(base, arch), _digest(context) + key = attempt.key = _key(container=container["key"], base_disk=base_identity, recipe=recipe, form="disk", + arch=arch, format=FORMAT) + store, ref, pinned = _stored(env, key) + if pinned is not None: + progress.at(pair, "reusing", "reusing the disk of the same key, made earlier") + return {"status": "unverified", "key": key, "image": pinned, "reused": True, "built_at": now()} + link = builds.link(platform, base.image()) + progress.at(pair, "starting its builder", f"making the disk {builder.where(link)}") + on = opened.enter_context(builds.open(link, platform, Path(work), base.image(), {})) + on.login(store.auth(container["image"])) + if base_disk is not None: + on.login(get_config().get_image_store().auth(base_disk)) + if (tools := published_pinned(DISK_TOOLS_IMAGE)) is None: + tools = f"agentenv-disk-tools:{DISK_TOOLS_IMAGE.tag()}" + on.build((context / "tools" / "Dockerfile").read_text(), tools, on.platform, {}, + context=f"{builder.CONTEXT}/tools") + dockerfile = (context / "Dockerfile").read_text() + if base_disk is None: + progress.at(pair, "making the base's disk") + base_disk = _made_base_disk(on, base, arch, f"{base_identity} {recipe}", tools, dockerfile) + progress.at(pair, "making the disk") + local = f"agentenv-computer:{key[7:23]}-disk-{arch}" + on.build(dockerfile, local, platform, {KEY_LABEL: key}, build_args={ + "ROOTFS": container["image"], "ROOTFS_CONFIG": _encoded(_config(on, container["image"], arch)), + "ESP": base_disk, "TOOLS": tools}) + attempt.stage = "push" + progress.at(pair, "pushing the disk") + on.push(local, ref, store) + return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, + "builder": on.name, "built_at": now()} + return attempt.result + + +def _made_base_disk(on: builder.Builder, base: ComputerArtifact, arch: str, made_from: str, tools: str, + dockerfile: str) -> str: + """The base's disk for ``arch``, made on ``on`` from the base's container with the boot recipe's ``dockerfile`` at + the base's size, under a tag of ``made_from``, what it's made from.""" + source, _ = _base(base, arch) + tag = f"agentenv-computer:{hashlib.sha256(made_from.encode()).hexdigest()[:16]}-base-disk-{arch}" + on.build(dockerfile, tag, f"linux/{arch}", {}, build_args={ + "ROOTFS": source, "ROOTFS_CONFIG": _encoded(_config(on, source, arch)), "TOOLS": tools, + "SIZE_GB": str(base.forms.disk.min_disk_gb)}) + return tag + + +@dataclass +class _Attempt: + """How far a pair's build got, its stage and key, and, when it failed, its result (``_attempting``).""" + + stage: str = "build" + key: str | None = None + result: dict | None = None + + +@contextlib.contextmanager +def _attempting(env: ComputerEnv, form: str, arch: str) -> Iterator[_Attempt]: + """A pair's build, whose failure becomes its result: skipped, or failed at the stage it reached, saying why.""" + attempt = _Attempt() try: - with tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened: - context, fetched, from_store = Path(work) / builder.CONTEXT, {}, {} - context.mkdir() - source, base_identity = _base(base, arch) - layer = _stage_driver(context, driver, base, from_store) - apps = [_stage_app(context, app, base, from_store) for app in env.composed_apps()] - layers = [layer, *apps] - wants = _wants(layers, _stage_protocol(context)) - platform, held = f"linux/{arch}", [] - - def on() -> builder.Builder: - if not held: - link = builds.link(platform) - native = platform in link.native_platforms() - progress.at(pair, "starting its builder", f"building natively {builder.where(link)}" if native else - f"building emulated, which is slow, {builder.where(link)}; " - f"{builder.instead_of_emulating(builds.provider, platform)}") - held.append(opened.enter_context(builds.open(link, platform, Path(work), base.image(), from_store))) - return held[0] - - inputs = packages.inputs_digest(wants, context, base_identity, arch) - lock = packages.stored(inputs, env.stored_id) if inputs is not None else None - if inputs is not None and lock is None: - stage = "resolve" - resolving = on() - progress.at(pair, "resolving packages") - lock, fetched = packages.resolve(inputs, wants, resolving, source, arch, env.stored_id) - progress.at(pair, "resolved", f"resolved {len(fetched)} files and stored them") - stage = "build" - dockerfile = _dockerfile(source, layers, base, _build_settings(env, base), lock) - key = "sha256:" + hashlib.sha256(json.dumps({ - "base": base_identity, "dockerfile": dockerfile, "context": _digest(context), - "lock": lock.digest() if lock is not None else None, "form": form, "arch": arch, - "format": FORMAT, - }, sort_keys=True).encode()).hexdigest() - store = get_config().get_image_store_for(env.stored_id) - store.ensure_repository(REPOSITORY) - ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) - if (pinned := _served(ref, store)) is not None: - progress.at(pair, "reusing", "reusing the image of the same key, built earlier") - kept = labels(pinned, arch, store.auth(pinned)) - return {"status": "unverified", "key": key, "image": pinned, "reused": True, - "emulated": kept.get(EMULATED_LABEL) == "true", "lock": inputs, - "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": _now()} - building_on = on() - progress.at(pair, "building the image") - if lock is not None: - packages.stage(lock, building_on, env.stored_id, fetched) - local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform - building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) - if reason := _compare(building_on, local, layers, pair, progress): - return _failed(env, form, arch, stage, key, reason) - installed = building_on.installed(local, layer.prefix) - building_on.build(f"FROM {local}\n", local, platform, { - INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", - }, context="label") - stage = "push" - progress.at(pair, "pushing the image") - building_on.push(local, ref, store) - return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, - "emulated": emulated, "builder": building_on.name, "lock": inputs, "installed": installed, - "built_at": _now()} + yield attempt except builder.Skipped as e: - return {"status": "skipped", "reason": str(e), "skipped_at": _now()} + attempt.result = {"status": "skipped", "reason": str(e), "skipped_at": now()} except builder.Unbuildable as e: - return _failed(env, form, arch, stage, key, str(e)) + attempt.result = failed(env, form, arch, attempt.stage, attempt.key, str(e)) except builder.ResolveError as e: - return _failed(env, form, arch, "resolve", None, str(e), log=e.log or None) + attempt.result = failed(env, form, arch, "resolve", None, str(e), log=e.log or None) except DockerBuildError as e: # Docker can list a platform it has lost the emulation for, until it restarts; the build says so. unemulated = "exec format error" in str(e) - return _failed(env, form, arch, stage, key, f"the builder can't run linux/{arch} programs: its Docker has no " - "emulation for it" if unemulated else "the build failed", log=str(e)) + attempt.result = failed(env, form, arch, attempt.stage, attempt.key, + f"the builder can't run linux/{arch} programs: its Docker has no emulation for it" + if unemulated else "the build failed", log=str(e)) except subprocess.CalledProcessError as e: - return _failed(env, form, arch, stage, key, f"{' '.join(e.cmd[:2])} failed: {(e.stderr or '').strip()}") + attempt.result = failed(env, form, arch, attempt.stage, attempt.key, + f"{' '.join(e.cmd[:2])} failed: {(e.stderr or '').strip()}") except (RuntimeError, ValueError, OSError) as e: - return _failed(env, form, arch, stage, key, f"{type(e).__name__}: {e}") + attempt.result = failed(env, form, arch, attempt.stage, attempt.key, f"{type(e).__name__}: {e}") + + +def _key(**inputs: Any) -> str: + """A build's key: a digest of everything that goes into it.""" + return "sha256:" + hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest() + + +def _stored(env: ComputerEnv, key: str) -> tuple[Any, str, str | None]: + """The env's image store, the ref there of the image of ``key``, and that ref pinned when the store serves one + already, for any env that built it.""" + store = get_config().get_image_store_for(env.stored_id) + store.ensure_repository(REPOSITORY) + ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) + return store, ref, _served(ref, store) + + +def _base_disk(base: ComputerArtifact, arch: str) -> tuple[str | None, str]: + """The base's disk for ``arch``, pinned to its digest, and its part of a disk's key: the base's disk form's image, + or the one published for the base's container sources; else None, and the base's container and disk size, which a + disk is made from with the boot recipe, the key's recipe.""" + if (image := base.disk_image()) is not None: + store = get_config().get_image_store() + pinned = image.pinned_name if "@" in image.pinned_name else pin_digest(image.pinned_name, + store.auth(image.pinned_name)) + return pinned, pinned + if (published := published_disk(base.image().source_digest or "")) is not None: + return published, published + return None, f"made from {_base(base, arch)[1]} at {base.forms.disk.min_disk_gb} GB" + + +def _config(on: builder.Builder, image: str, arch: str) -> dict: + """The config object of ``image``'s ``arch`` image: from its registry, or from the builder's Docker for one only it + holds.""" + if names_registry(image): + return image_config(image, arch, get_config().get_image_store().auth(image)) + return on.image_config(image) + + +def _encoded(config: dict) -> str: + return base64.b64encode(json.dumps(config, sort_keys=True).encode()).decode() def _compare(building_on: builder.Builder, tag: str, layers: list[_Layer], pair: str, @@ -439,16 +522,10 @@ def _protocol_source() -> Path | None: def _build_settings(env: ComputerEnv, base: ComputerArtifact) -> dict[str, Any]: """The env's values of the base's build-phase settings, which the composed image fixes.""" - values = _values(env) + values = env.computer_values() return {name: values[name] for name, setting in base.config.items() if setting.phase == "build" and name in values} -def _values(env: ComputerEnv) -> dict[str, Any]: - """The env's values of its base's settings: its ``computer_settings``, and its display as the resolution.""" - display = env.display - return {**env.computer_settings, **({"resolution": f"{display['width']}x{display['height']}"} if display else {})} - - def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build_settings: dict, lock: packages.Lock | None) -> str: """The composed image's Dockerfile but its key's label, which the key is a digest of: the build-phase settings, the @@ -527,47 +604,42 @@ def _served(ref: str, store) -> str | None: return None -def _failed(env: ComputerEnv, form: str, arch: str, stage: str, key: str | None, reason: str, - log: str | None = None) -> dict: - result = {"status": "failed", "stage": stage, "reason": reason, "failed_at": _now()} - if key is not None: - result["key"] = key - if log is not None: - stored = FileArtifact.put_bytes(derive_id(env.stored_id, f"build_log_{form}_{arch}"), description=( - f"the {form}/{arch} build of {env.stored_id} v{env.version}, which failed"), filename="build.log", - content=log.encode(), content_type="text/plain") - result["log"] = {"id": stored.id, "version": stored.version} - result["reason"] = f"{reason}: {log.strip().splitlines()[-1] if log.strip() else 'no output'}" - return result - - -def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str]) -> dict | None: +def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str], disks: dict[str, str]) -> dict | None: """Write the env's computer over the images ``built``, by arch, joined into one index in the env's image store, signed in to it: its arches those, its OS and session the base's, its settings the base's with the env's values as their - defaults. None when nothing built.""" + defaults; with a disk form over ``disks`` when there's one for each of those arches. None when nothing built.""" if not built: return None arches = [arch for arch in base.arch if arch in built] - store = get_config().get_image_store_for(env.stored_id) - index = store.image_ref(REPOSITORY, "index-" + hashlib.sha256( - ",".join(built[arch] for arch in arches).encode()).hexdigest()) - write_index(index, {f"linux/{arch}": built[arch] for arch in arches}, store.auth(index)) - image = DockerImageArtifact.put_ref(derive_id(env.stored_id, "computer_image"), description=( - f"{env.stored_id}'s composed computer"), image_name=index) + image = _joined(env, "computer_image", "composed computer", built, arches) config = {name: setting.model_dump() for name, setting in base.config.items()} - for name, value in _values(env).items(): + for name, value in env.computer_values().items(): config[name]["default"] = value + forms = {"container": {"image": {"id": image.id, "version": image.version}, "init": base.forms.container.init, + "input_channel": base.forms.container.input_channel}} + if base.forms.disk is not None and all(arch in disks for arch in arches): + disk = _joined(env, "computer_disk", "composed computer's disk", disks, arches) + forms["disk"] = {**base.forms.disk.model_dump(exclude={"image"}), "image": {"id": disk.id, "version": disk.version}} computer = ComputerArtifact.put( id=derive_id(env.stored_id, "computer"), description=f"{base.id} composed with {env.driver.id} for {env.stored_id}", - arch=arches, os=base.os.model_dump(), session=base.session.model_dump(), config=config, - forms={"container": {"image": {"id": image.id, "version": image.version}, "init": base.forms.container.init, - "input_channel": base.forms.container.input_channel}}) - composed = env.metadata.get("composed", {}).get("container", {}) - if probes := {arch: composed[arch]["probe"] for arch in arches if "probe" in composed.get(arch, {})}: - computer.merge_metadata({"validated_boot": {"container": probes}}) + arch=arches, os=base.os.model_dump(), session=base.session.model_dump(), config=config, forms=forms) + composed = env.metadata.get("composed", {}) + if probes := {form: by_arch for form in forms if (by_arch := { + arch: composed[form][arch]["probe"] for arch in arches if "probe" in composed.get(form, {}).get(arch, {})})}: + computer.merge_metadata({"validated_boot": probes}) return {"id": computer.id, "version": computer.version} +def _joined(env: ComputerEnv, name: str, what: str, images: dict[str, str], arches: list[str]) -> DockerImageArtifact: + """``images`` of ``arches`` joined into one index in the env's image store, signed in to it, as an image artifact.""" + store = get_config().get_image_store_for(env.stored_id) + index = store.image_ref(REPOSITORY, "index-" + hashlib.sha256( + ",".join(images[arch] for arch in arches).encode()).hexdigest()) + write_index(index, {f"linux/{arch}": images[arch] for arch in arches}, store.auth(index)) + return DockerImageArtifact.put_ref(derive_id(env.stored_id, name), description=f"{env.stored_id}'s {what}", + image_name=index) + + def _digest(folder: Path) -> str: """A digest of the files under ``folder``, by their paths, bytes and whether each is executable, which the composer alone decides (umask never sets it).""" @@ -578,53 +650,6 @@ def _digest(folder: Path) -> str: return "sha256:" + hashed.hexdigest() -def _now() -> str: - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -class _Record: - """The env's ``composed`` metadata, by form and arch. Each result is merged into the stored table pair by pair, the - table read again when another build wrote it first, so builds of one env at once keep each other's pairs, and a - result never undoes one that got further (``_rank``).""" - - def __init__(self, env: ComputerEnv): - self.env, self.lock = env, threading.Lock() - - def started(self, pairs: list[tuple[str, str]]) -> None: - now = _now() - self._merge({pair: {"status": "building", "started_at": now} for pair in pairs}) - - def finished(self, form: str, arch: str, result: dict) -> None: - self._merge({(form, arch): result}) - - def built(self) -> dict[str, str]: - """Each arch's image of the container form, of the pairs built or reused.""" - return {arch: result["image"] for arch, result in self.env.metadata.get("composed", {}).get("container", {}).items() - if result.get("status") in _BUILT} - - def computer(self, ref: dict | None) -> None: - """Record the env's computer, with the images it was written over.""" - if ref is not None: - with self.lock: - self.env.merge_metadata({"computer": {**ref, "images": self.built()}}) - - def _merge(self, results: dict[tuple[str, str], dict]) -> None: - with self.lock: - for attempt in range(_WRITE_TRIES): - composed = copy.deepcopy(self.env.metadata.get("composed", {})) - for (form, arch), result in results.items(): - stored = composed.setdefault(form, {}).get(arch, {}) - if _rank(result) >= _rank(stored): - composed[form][arch] = {"started_at": stored.get("started_at"), **result} - try: - self.env.update_metadata({**self.env.metadata, "composed": composed}) - return - except ConcurrentModificationError: - if attempt == _WRITE_TRIES - 1: - raise - self.env.metadata = Env.get(self.env.stored_id, self.env.version).metadata - - class _Progress: """What a compose reports as it goes, to ``on_progress`` and the log: a line as each pair reaches a stage, one each ``_HEARTBEAT_S`` a stage goes on, and how each pair ended, with the time since the compose began.""" @@ -679,11 +704,3 @@ def _beat(self) -> None: def _took(seconds: float) -> str: minutes, seconds = divmod(round(seconds), 60) return f"{minutes} min {seconds} s" if minutes else f"{seconds} s" - - -def _rank(result: dict) -> int: - """How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an - image. A result takes a stored one's place only when it got at least as far.""" - if result.get("status") == "ready": - return 2 - return 1 if result.get("status") == "unverified" or result.get("stage") == "probe" else 0 diff --git a/src/agent_env/env/envs/computer/utils/probing.py b/src/agent_env/env/envs/computer/utils/probing.py new file mode 100644 index 000000000..c5df75333 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/probing.py @@ -0,0 +1,90 @@ +"""Gate 3: a computer env's build booted on a provider that runs it natively and probed at ``full``, by its compose or, +for a build its compose couldn't boot, on its first deploy.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from agentenv_protocol.computer.conformance import run_probe + +from agent_env.artifact.artifacts.file import FileArtifact +from agent_env.env.env import Env +from agent_env.env.envs.computer.utils import boot +from agent_env.env.envs.computer.utils.composed import Record, failed, now +from agent_env.store.ids import derive_id +from agent_env.store.local_state import awaiting_lock + +if TYPE_CHECKING: + from agentenv_protocol.computer.conformance import ProbeReport + + from agent_env.env.envs.computer.driver_env import ComputerDriverEnv + from agent_env.env.envs.computer.env import ComputerEnv + from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider + +#: How long a probe's computer may run before its sandbox provider stops it. +_PROBE_TTL_S = 1800 +#: How long the probe at full may take on a computer that booted: about ten seconds on a healthy one. +_PROBE_TIMEOUT_S = 120 + + +async def probe_first_boot(env: ComputerEnv, form: str, arch: str, sandbox_provider: SandboxProvider) -> dict: + """Gate 3 for a build its compose couldn't probe, on its first boot on ``sandbox_provider``, which boots it + natively: the result, recorded. Deploys of one build at once, in this process or another, probe it once: each + waits for the one probing, then takes the result it recorded, a failure's too, which keeps the build's key.""" + build = env.metadata["composed"][form][arch] + async with awaiting_lock(f"computer-first-boot:{env.stored_id}:{form}:{arch}:{build['image']}"): + recorded = (await asyncio.to_thread(Env.get, env.stored_id, env.version)).metadata["composed"][form][arch] + if recorded.get("key") == build["key"] and recorded["status"] != "unverified": + return recorded + result = await checked([sandbox_provider], env, env.driver, form, arch, recorded) + await asyncio.to_thread(Record(env).finished, form, arch, result) + return result + + +async def checked(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, + result: dict) -> dict: + """The build of ``result`` booted on the first of ``links`` that boots it and probed at ``full``: ready, with the + driver's card, or failed at the probe; each with a summary of the probe and its report stored.""" + try: + report, card, provider = await _probe(links, env, driver, result["image"], form) + except TimeoutError: + return failed(env, form, arch, "probe", result["key"], f"the probe didn't finish within {_PROBE_TIMEOUT_S} s") + except Exception as e: + return failed(env, form, arch, "probe", result["key"], f"it didn't boot: {e}") + stored = FileArtifact.put_bytes(derive_id(env.stored_id, f"probe_report_{form}_{arch}"), description=( + f"the {form}/{arch} probe of {env.stored_id} v{env.version}"), filename="probe.json", + content=report.model_dump_json(indent=1).encode(), content_type="application/json") + probe = {"level": report.level, "provider": provider, "passed": report.passed, "checks": report.counts(), + "extensions": report.extensions, "report": {"id": stored.id, "version": stored.version}, + "probed_at": now()} + if not report.passed: + failing = "; ".join(f"{c.check} {c.name}: {c.detail}" for c in report.checks if c.status == "failed") + return {**result, "status": "failed", "stage": "probe", "reason": f"the probe failed: {failing}", + "failed_at": now(), "probe": probe} + return {**result, "status": "ready", "card": card, "probe": probe} + + +async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: ComputerDriverEnv, image: str, + form: str = "container") -> tuple[ProbeReport, dict, str]: + """``image``, of ``form``, booted at the env's settings and its apps' defaults with a control token of its own, as + a deploy with no ``env_params`` boots it, on the first of ``links`` that boots it, probed at ``full`` with that + token within ``_PROBE_TIMEOUT_S`` and torn down; the report, the driver's card and the provider's name.""" + token = boot.control_token() + environment = {**boot.settings_environment(env.computer, env.computer_values(), environment_name=env.environment_name), + **boot.app_settings_environment(env.composed_apps(), env.app_defaults(), env.computer.os.type)} + problems = [] + for link in links: + try: + booted = await boot.boot(link, image=image, port=driver.port, environment=environment, token=token, + network_policy=None, timeout=_PROBE_TTL_S, attribution=None, + ready_s=boot.ready_timeout(env.composed_apps(), env.computer.os.type), form=form) + except Exception as e: + problems.append(f"{type(link).__name__}: {e}") + continue + try: + async with asyncio.timeout(_PROBE_TIMEOUT_S): + return await run_probe(booted.url, "full", token=token), booted.card, type(link).__name__ + finally: + await boot.discard(booted.sandbox) + raise RuntimeError("; ".join(problems)) diff --git a/src/agent_env/env/published_images.py b/src/agent_env/env/published_images.py index cf7d4ad1e..54a426c05 100644 --- a/src/agent_env/env/published_images.py +++ b/src/agent_env/env/published_images.py @@ -1,13 +1,16 @@ -"""The images agent-env publishes for the infra envs, the agents it ships, an example's env and its base computers, for -linux/amd64 and linux/arm64: one per Dockerfile, built from the Dockerfile's folder. Each is tagged by its build context's content, so -an install holding the same sources finds the image built from them; a release wheel also records the digest its -release published under each tag.""" +"""The images agent-env publishes for the infra envs, the agents it ships, an example's env, its base computers and their +disk forms, for linux/amd64 and linux/arm64: one per Dockerfile, built from the Dockerfile's folder. Each is tagged by +its build context's content, and an image built over other published images by theirs too, so an install holding the +same sources finds the image built from them; a release wheel also records the digest its release published under each +tag.""" from __future__ import annotations +import hashlib import json import logging import re +import tomllib from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -28,24 +31,34 @@ _AGENTS = Path(__file__).parents[1] / "agents" / "agents" _EXAMPLES = Path(__file__).parents[1] / "examples" _COMPUTERS = Path(__file__).parents[1] / "computers" / "artifacts" +# The Linux disk form's boot recipe, disk tools and QEMU launcher. +DISK = Path(__file__).parents[1] / "computers" / "disk" / "linux" @dataclass(frozen=True) class PublishedImage: """An image agent-env publishes, under ``name`` in the registry, built from ``dockerfile`` with its folder as the - build context.""" + build context, with each of ``args``, a build arg its Dockerfile builds from, set to that published image, and each + of ``build_args`` set to its value.""" name: str dockerfile: Path + args: tuple[tuple[str, PublishedImage], ...] = () + build_args: tuple[tuple[str, str], ...] = () @property def repository(self) -> str: return f"{REGISTRY}/{self.name}" def tag(self) -> str: - """``src-`` and the hex of the build context's source digest, the same for the same sources on any platform.""" + """``src-`` and the hex of the build context's source digest, with each of ``args``' tags and ``build_args``' + values when it has any: the same for the same sources on any platform.""" digest = BuildContext.of(self.dockerfile.parent, self.dockerfile).source_digest(None) - return f"src-{digest.removeprefix('sha256:')}" + if self.args or self.build_args: + digest = hashlib.sha256("\0".join([digest.removeprefix("sha256:"), + *(f"{arg}={image.tag()}" for arg, image in self.args), + *(f"{arg}={value}" for arg, value in self.build_args)]).encode()).hexdigest() + return source_tag(digest) def ref(self) -> str: """The repository and the tag of the image built from these sources.""" @@ -61,6 +74,20 @@ def ref(self) -> str: COMPUTER_IMAGES = (PublishedImage("agentenv-ubuntu-24.04", _COMPUTERS / "ubuntu-24.04" / "Dockerfile"),) # The shipped bundles' published images, whose requirements keep a protocol floor. BUNDLE_IMAGES = AGENT_IMAGES + EXAMPLE_IMAGES + COMPUTER_IMAGES +# What makes and boots a Linux disk form: the disk tools a build runs on its own platform, and the QEMU launcher. +DISK_TOOLS_IMAGE = PublishedImage("agentenv-disk-tools", DISK / "tools" / "Dockerfile") +LAUNCHER_IMAGE = PublishedImage("agentenv-qemu-launcher", DISK / "launcher" / "Dockerfile") +# Each base computer's disk form: its container form's root filesystem on the boot recipe, at the size its [forms.disk] +# declares, so a run pulls it rather than building it from the two. +DISK_IMAGES = tuple(PublishedImage( + f"{image.name}-disk", DISK / "boot" / "Dockerfile", (("ROOTFS", image), ("TOOLS", DISK_TOOLS_IMAGE)), + (("SIZE_GB", str(tomllib.loads((image.dockerfile.parent / "artifact.toml").read_text())["forms"]["disk"][ + "min_disk_gb"])),)) for image in COMPUTER_IMAGES) + + +def source_tag(digest: str) -> str: + """The tag of a published image whose sources have the digest ``digest``.""" + return f"src-{digest.removeprefix('sha256:')}" def published_digest(image: PublishedImage) -> str | None: @@ -70,6 +97,19 @@ def published_digest(image: PublishedImage) -> str | None: return release_digests(RELEASE_FILE).get(ref) or _registry_digest(ref) +def published_pinned(image: PublishedImage) -> str | None: + """``image``'s ref pinned to the digest it's published under for the sources installed, or None.""" + digest = published_digest(image) + return f"{image.ref()}@{digest}" if digest else None + + +def published_disk(sources: str) -> str | None: + """The pinned disk form published for the shipped base computer whose container image's installed build context has + the source digest ``sources``, or None when no shipped base has those sources or its disk isn't published.""" + return next((published_pinned(disk) for disk in DISK_IMAGES if dict(disk.args)["ROOTFS"].tag() == source_tag(sources)), + None) + + def published_bundle_image(sources: str) -> tuple[str, str] | None: """The ref and digest of the image published for a shipped bundle's image (an agent's, an example env's or a base computer's) whose @@ -77,9 +117,8 @@ def published_bundle_image(sources: str) -> tuple[str, str] | None: shipped image has those sources or the registry doesn't serve it anonymously, as a sandbox pulls it: a new package stays private until a package admin makes it public, and a sandbox running an image by name can't build it instead.""" - tag = f"src-{sources.removeprefix('sha256:')}" for image in BUNDLE_IMAGES: - if image.tag() == tag: + if image.tag() == source_tag(sources): ref = image.ref() if (served := _registry_digest(ref)) is None: return None diff --git a/src/agent_env/providers/env_providers/env_computer_provider.py b/src/agent_env/providers/env_providers/env_computer_provider.py index bd7ca5c34..049592011 100644 --- a/src/agent_env/providers/env_providers/env_computer_provider.py +++ b/src/agent_env/providers/env_providers/env_computer_provider.py @@ -14,12 +14,14 @@ from agent_env.attribution import Attribution from agent_env.env.env import DeployedSandboxEnv, EnvCapabilityUnsupported from agent_env.env.envs.computer.utils import params as computer_params +from agent_env.env.envs.computer.utils.probing import probe_first_boot from agent_env.env.envs.computer.utils.boot import ( app_settings_environment, boot, control_environment, control_token, discard, + native_links, ready, ready_timeout, recorded_app_settings, @@ -71,7 +73,7 @@ async def deploy( params, values, problems = computer_params.checked(env, env_params) if problems: raise DeployRefused(f"{env.id}: " + "; ".join(problems)) - if not (builds := env.ready_builds(params.form)): + if not (builds := env.deployable_builds(params.form)): raise DeployRefused(f"{env.id} has no ready build to deploy; its builds are {env.build_statuses() or 'none yet'}") attempt = functools.partial(self._boot, env, builds, params, values, ttl_seconds=ttl_seconds, disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, attribution=attribution) @@ -87,12 +89,17 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par """One attempt on one sandbox provider: the first build it runs natively, booted and readied; its record.""" from agent_env.env.envs.computer.env import ComputerEnv - native = [(form, arch, build) for form, arch, build in builds - if f"linux/{arch}" in sandbox_provider.native_platforms()] + native = [(form, arch, build) for form, arch, build in builds if native_links(sandbox_provider, arch, form)] if not native: raise ValueError(f"{type(sandbox_provider).__name__} runs none of {env.id}'s ready builds natively " f"({', '.join(f'{form}/{arch}' for form, arch, _ in builds)})") form, arch, build = native[0] + if refusal := env.disk_size_refusal(form, disk_size_gb): + raise ValueError(refusal) + if build["status"] == "unverified": + build = await probe_first_boot(env, form, arch, sandbox_provider) + if build["status"] != "ready": + raise RuntimeError(f"{env.id}'s {form}/{arch} failed its probe on its first boot: {build['reason']}") computer, name, token = env.computer_artifact(), env.environment_name, control_token() values = _values(computer, values) app_settings = recorded_app_settings(env.composed_apps(), computer_params.app_settings(env, params.app_settings), @@ -102,7 +109,7 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par sandbox_provider, image=build["image"], port=env.driver.port, network_policy=policy, token=token, environment=_environment(env, computer, values, params), timeout=ttl_seconds, attribution=attribution, disk_size_gb=disk_size_gb, cpu=cpu, memory_mb=memory_mb, - ready_s=ready_timeout(env.composed_apps(), env.computer.os.type)) + ready_s=ready_timeout(env.composed_apps(), env.computer.os.type), form=form) self._container_sandboxes.append(booted.sandbox) self._environment_sandboxes[name] = booted.sandbox try: @@ -128,9 +135,9 @@ async def _boot(self, env: ComputerEnv, builds: list[tuple[str, str, dict]], par return record async def reboot(self, env: ComputerEnv, deployed: DeployedSandboxEnv) -> None: - """Replace the computer ``deployed`` records with a fresh container of the image it booted, once its recordings - ended, its settings applied as that deploy applied them (a generated one minted afresh) and its control token - the record's, on the same sandbox and port, readied as a deploy readies it.""" + """Replace the computer ``deployed`` records with a fresh container of the image it booted, or its disk booted + afresh, once its recordings ended, its settings applied as that deploy applied them (a generated one minted + afresh) and its control token the record's, on the same sandbox and port, readied as a deploy readies it.""" record = deployed.metadata["computer"] params, values, _ = computer_params.checked(env, record["env_params"]) computer = ComputerArtifact.get(record["computer"]["id"], record["computer"]["version"]) @@ -138,10 +145,13 @@ async def reboot(self, env: ComputerEnv, deployed: DeployedSandboxEnv) -> None: await end_recordings(deployed, "reset") sandbox_provider = build_sandbox_provider(deployed.sandbox_type) sandbox = await sandbox_provider.get_sandbox(deployed.sandbox_id) + environment = {**_environment(env, computer, values, params), **control_environment(deployed.control_token)} try: - await sandbox_provider.reset_container(sandbox, image_name=record["image"], port=env.driver.port, - env={**_environment(env, computer, values, params), - **control_environment(deployed.control_token)}) + if record.get("form") == "disk": + await sandbox_provider.reset_vm(sandbox, image=record["image"], env=environment) + else: + await sandbox_provider.reset_container(sandbox, image_name=record["image"], port=env.driver.port, + env=environment) except NotImplementedError as e: # a reset the env supports, which this provider can't carry out raise RuntimeError(f"'{env.id}' can't be reset on {type(sandbox_provider).__name__}: {e}") from None card = await ready(deployed.environment_url, ready_timeout(env.composed_apps(), env.computer.os.type), diff --git a/src/agent_env/providers/sandbox_providers/AGENTS.md b/src/agent_env/providers/sandbox_providers/AGENTS.md index e25a2f38b..548bd5c56 100644 --- a/src/agent_env/providers/sandbox_providers/AGENTS.md +++ b/src/agent_env/providers/sandbox_providers/AGENTS.md @@ -50,13 +50,15 @@ with `exposed_ports=[port]` and returns the VM, and the deploy then loads and st | Method | Required | Called by | |---|---|---| -| `create_vm(*, image, boot_mode, cpu, memory, disk_size_gb, timeout, exposed_ports, setup_for_gateway, attribution, network_policy) -> VmSandbox` | VM backend | env deploys, `deploy_sandbox` in VM mode, `create_container`, and your `create_sandbox` | +| `create_vm(*, image, boot_mode, cpu, memory, disk_size_gb, timeout, exposed_ports, setup_for_gateway, attribution, network_policy, env) -> VmSandbox` | VM backend | env deploys, `deploy_sandbox` in VM mode, `create_container`, and your `create_sandbox`; with a computer's disk form as `image` (an image holding `/disk/disk.img`) and its `env`, a computer env's deploy and gate 3, on a provider whose `disk_image_platforms()` names the disk's platform: boot the disk, `env` in its NoCloud seed (`computers/disk/linux/launcher/` boots one under QEMU), reached on `exposed_ports`, with no exec channel into it | | `create_sandbox(*, image_name, port, env, cpu, memory, disk_size_gb, timeout, attribution, network_policy) -> Sandbox` | yes (a VM backend wraps `create_vm`) | an agent's sandbox (`provider.create_sandbox`); `create_container` of a container-only backend | | `get_sandbox(sandbox_id) -> Sandbox` | yes | every step after the one that created the sandbox, and teardown | | `terminate_sandbox(sandbox_id)` | no | teardown. The default calls `get_sandbox` then `terminate()`, and treats `SandboxNotFoundError` as done; override when you can terminate without a live handle | | `create_container(...)` | inherited | containers that run an env or agent image. Override only when the platform boots images natively | | `reset_container(sandbox, *, image_name, port, env)` | inherited | a computer env's reset, which replaces the container `create_container` ran with a fresh one on the same sandbox and port. The default works on a VM backend; one whose sandbox is the container itself raises `NotImplementedError` | | `native_platforms()` | no | a computer env's deploy and gate 3, which boot a build only on a provider that runs its arch without emulation. The default is `linux/amd64`; local Docker returns this machine's | +| `disk_image_platforms()` | no | the same for a computer's disk form: the platforms whose disks `create_vm` boots without emulation. The default is none; local Docker returns this machine's where it can pass `/dev/kvm` to a container | +| `reset_vm(sandbox, *, image, env)` | no | a computer env's reset of a disk `create_vm` booted: boot it afresh on the same sandbox and ports. The default raises `NotImplementedError` | | `prepare_image(image, *, attribution)` | no | before a sandbox is asked for an image, outside any creation deadline; build a context-only image here, and pull one with an `image_digest` by it, building it from its context when it can't be pulled | | `from_config(**config)` | no | receives `[sandbox.providers..config]` with `secret:` references resolved. Override to fail early with a `ConfigError` that names the missing key (`boxd` also rejects unknown keys) | | `close()` | no | release SDK clients | diff --git a/src/agent_env/providers/sandbox_providers/local_docker/disk.py b/src/agent_env/providers/sandbox_providers/local_docker/disk.py new file mode 100644 index 000000000..a667bb367 --- /dev/null +++ b/src/agent_env/providers/sandbox_providers/local_docker/disk.py @@ -0,0 +1,62 @@ +"""Booting a computer's disk form on this machine's Docker: the QEMU launcher (the published image for the sources +installed, else built here from its folder) with the disk copied in, as an image this machine keeps for the next boot, +run with /dev/kvm.""" + +import asyncio +import base64 +import hashlib +import math +import os +import subprocess +import tempfile +from pathlib import Path, PurePosixPath + +from agent_env.artifact.artifacts.computer import DISK_FILE +from agent_env.env.published_images import LAUNCHER_IMAGE, published_pinned +from agent_env.providers.sandbox_providers.host_images import host_platform, present +from agent_env.utils.docker.build import build_image + +KVM = "/dev/kvm" +#: How the launcher takes the computer's environment, each variable under this prefix, base64'd (see its launch script). +ENV_PREFIX = "VM_ENV_" + + +def kvm_platforms() -> frozenset[str]: + """This machine's platform when its Docker can pass KVM to a container: /dev/kvm here, readable and writable.""" + return frozenset({host_platform()}) if os.access(KVM, os.R_OK | os.W_OK) else frozenset() + + +async def machine_image(disk: str) -> str: + """An image of the launcher holding ``disk``'s ``DISK_FILE`` for this machine's platform, built unless held.""" + launcher = await asyncio.to_thread(_launcher) + tag = "agentenv-vm:" + hashlib.sha256(f"{launcher} {disk}".encode()).hexdigest()[:16] + if not await asyncio.to_thread(present, tag): + folder = f"/{PurePosixPath(DISK_FILE).parent}/" + await asyncio.to_thread(_build, f"FROM {launcher}\nCOPY --from={disk} {folder} {folder}\n", tag) + return tag + + +def launcher_args(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> list[str]: + """The ``docker run`` arguments that size the launcher's VM, forward ``ports`` and hand it ``env``, each value + base64'd, so one of several lines reaches the computer whole.""" + variables = {"VM_PORTS": " ".join(map(str, ports)), "VM_CPUS": str(max(1, math.ceil(cpu))), + "VM_MEMORY_MB": str(memory), + **{f"{ENV_PREFIX}{name}": base64.b64encode(value.encode()).decode() for name, value in env.items()}} + return ["--device", KVM, *(arg for name, value in variables.items() for arg in ("-e", f"{name}={value}"))] + + +def _launcher() -> str: + if (pinned := published_pinned(LAUNCHER_IMAGE)) is not None: + if not present(pinned): + subprocess.run(["docker", "pull", "-q", pinned], check=True, capture_output=True, text=True) + return pinned + tag = f"agentenv-qemu-launcher:{LAUNCHER_IMAGE.tag()}" + if not present(tag): + build_image(LAUNCHER_IMAGE.dockerfile, LAUNCHER_IMAGE.dockerfile.parent, tag, platform=None) + return tag + + +def _build(dockerfile: str, tag: str) -> None: + with tempfile.TemporaryDirectory(prefix="agentenv-vm-") as context: + (Path(context) / "Dockerfile").write_text(dockerfile) + build_image(Path(context) / "Dockerfile", Path(context), tag, platform=None) diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index a6c1dd74f..84c954c0d 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -39,6 +39,7 @@ from agent_env.attribution import Attribution from agent_env.config.paths import state_root from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here +from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_image from agent_env.providers.sandbox_providers.local_docker.relay import relay_image from agent_env.providers.sandbox_providers.sandbox import ( SANDBOX_LABEL, @@ -115,6 +116,8 @@ def trust_env(directory: str) -> dict[str, str]: _PORT_MAP_FILE = ".port-map.json" # The internal network a container with no network runs on, so a sandbox rebuilt from its id removes it and its relay. _CLOSED_NETWORK_FILE = ".closed-network" +# The size of the VM a disk booted on, so a reset boots it at the same size. +_VM_SIZE_FILE = ".vm-size.json" # The host ports sandboxes publish on: below the default ranges outgoing connections take their local ports from (32768 # up on Linux, the VM Docker Desktop or Rancher Desktop runs Docker in included; 49152 up on macOS and Windows) and # below Kubernetes' node ports (30000-32767). @@ -657,6 +660,10 @@ def url_from_sandbox(self, url: str) -> str: def native_platforms(cls) -> frozenset[str]: return frozenset({host_platform()}) + @classmethod + def disk_image_platforms(cls) -> frozenset[str]: + return kvm_platforms() + @classmethod def supports_network_policy(cls, policy: NetworkPolicy) -> bool: """Unrestricted, or a container with no network at all (``create_container``).""" @@ -675,10 +682,15 @@ async def create_vm( setup_for_gateway: bool = True, attribution: Optional[Attribution] = None, network_policy: Optional[NetworkPolicy] = None, + env: Optional[dict[str, str]] = None, ) -> LocalSandbox: + """This machine as a VM, or with ``image``, a computer's disk form booted under KVM in the QEMU launcher's + container, reached on each of ``exposed_ports``, at the disk's own size: ``disk_size_gb`` is this machine's.""" refuse_unenforceable_policy(self, network_policy) if network_policy is not None and network_policy.restricts_egress: raise NetworkPolicyUnsupportedError(f"{type(self).__name__} takes a container's network away, not a VM's") + if image is not None and host_platform() not in self.disk_image_platforms(): + raise ValueError(f"{type(self).__name__} boots a disk only where this machine's Docker can pass it /dev/kvm") # Allocated here rather than in the sandbox constructor: probing for a free port # is I/O, and callers that only need a handle must not pay for it. Off the event # loop, since a draw waits for any other process's draw to finish. @@ -687,8 +699,40 @@ async def create_vm( (sandbox.work_dir / _PORT_MAP_FILE).write_text(json.dumps(sandbox._port_map)) sandbox.network_policy = self.effective_network_policy(network_policy) sandbox.prepared = frozenset(self._prepared) + if image is not None: + (sandbox.work_dir / _VM_SIZE_FILE).write_text(json.dumps({"cpu": cpu, "memory": memory})) + try: + await self._boot_disk(sandbox, image=image, env=env or {}) + (sandbox.work_dir / _CONTAINER_MODE_MARKER).write_text(sandbox.container_name) + except BaseException: + with contextlib.suppress(Exception): + await sandbox.terminate() + await asyncio.to_thread(remove_local_work_dir, sandbox.sandbox_id) + raise return sandbox + async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None: + await sandbox.exec_script(f"docker rm -f {shlex.quote(sandbox.container_name)} > /dev/null") + await self._boot_disk(sandbox, image=image, env=env) + + async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str, str]) -> None: + """The launcher booting ``image`` as the sandbox's container, its ports published as a container's are.""" + size = json.loads((sandbox.work_dir / _VM_SIZE_FILE).read_text()) + await sandbox.pull_images([image]) + machine = await machine_image(image) + ports = list(sandbox._port_map) + publish = [arg for port in ports for spec in port_bindings(sandbox.host_ips, sandbox.host_port(port), port) + for arg in ("-p", spec)] + running = asyncio.ensure_future(asyncio.to_thread( + _docker, "run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", + *publish, *launcher_args(ports, env, size["cpu"], size["memory"]), machine)) + try: + await asyncio.shield(running) + except asyncio.CancelledError: # the container docker is still starting must exist before teardown removes it + with contextlib.suppress(Exception): + await running + raise + async def create_container( self, *, diff --git a/src/agent_env/providers/sandbox_providers/sandbox_provider.py b/src/agent_env/providers/sandbox_providers/sandbox_provider.py index 2ee4dfaef..4246ad97d 100644 --- a/src/agent_env/providers/sandbox_providers/sandbox_provider.py +++ b/src/agent_env/providers/sandbox_providers/sandbox_provider.py @@ -90,7 +90,11 @@ async def create_vm( setup_for_gateway: bool = True, attribution: Optional[Attribution] = None, network_policy: Optional[NetworkPolicy] = None, + env: Optional[dict[str, str]] = None, ) -> VmSandbox: + """A VM. A provider whose ``disk_image_platforms`` names a platform boots a computer's disk form, ``image``, an + image holding ``computer.DISK_FILE``, for it, its ``env`` in the disk's NoCloud seed; such a sandbox has no exec + channel.""" raise NotImplementedError(f"{type(self).__name__} does not support create_vm") async def create_container( @@ -154,6 +158,11 @@ async def reset_container(self, sandbox: Sandbox, *, image_name: str, port: int, await sandbox.exec_script(f"docker rm -f {shlex.quote(sandbox.container_name)} > /dev/null") await self._start_container(sandbox, image_name=image_name, port=port, env=env) + async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None: + """Boot the disk create_vm booted on ``sandbox`` afresh, from ``image`` with ``env``, reached on its ports as + before: the sandbox, its id and its URLs stay.""" + raise NotImplementedError(f"{type(self).__name__} can't boot a disk afresh in place") + async def prepare_image(self, image: DockerImageArtifact, *, attribution: Optional[Attribution] = None) -> None: """Get ready to run ``image`` by its name, before create_sandbox or create_container is asked for it and outside any deadline on creating one, for the deploy ``attribution`` describes, as it describes the sandbox. Does @@ -316,6 +325,11 @@ def native_platforms(cls) -> frozenset[str]: """The platforms its containers run without emulation.""" return frozenset({DEFAULT_BUILD_PLATFORM}) + @classmethod + def disk_image_platforms(cls) -> frozenset[str]: + """The platforms whose disk images (a computer's disk form) its create_vm boots without emulation: none here.""" + return frozenset() + @classmethod def effective_network_policy(cls, policy: Optional[NetworkPolicy]) -> NetworkPolicy: """``policy`` with the platform's own hosts unioned in; identity unless it allowlists.""" diff --git a/src/agent_env/store/image_store/image_contents.py b/src/agent_env/store/image_store/image_contents.py index 12f325a9f..d48aff834 100644 --- a/src/agent_env/store/image_store/image_contents.py +++ b/src/agent_env/store/image_store/image_contents.py @@ -1,21 +1,39 @@ -"""What an image a registry serves holds, read without pulling it: the image manifest each Linux architecture runs, and -an image's os-release, from the lowest layer that holds it.""" +"""What an image a registry serves holds, read without pulling it: the image manifest each Linux architecture runs, an +image's config, its os-release, from the lowest layer that holds it, and a disk form's disk, from the start of its +layer.""" from __future__ import annotations +import io import json +import struct import tarfile import tempfile +import zlib +from dataclasses import dataclass from pathlib import Path, PurePosixPath from agent_env.store.image_store.oci_registry_credentials import RegistryAuth -from agent_env.store.image_store.registry_api import pin_digest, read_blob, read_manifest +from agent_env.store.image_store.registry_api import pin_digest, read_blob, read_blob_start, read_manifest _INDEX_TYPES = ("application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json") # os-release(5): /etc/os-release first, else /usr/lib/os-release; on Ubuntu the first links to the second. _OS_RELEASE = ("etc/os-release", "usr/lib/os-release") # tarfile reads zstd from Python 3.14. _READS_ZSTD = "zst" in tarfile.TarFile.OPEN_METH +# How much of a disk layer is read: its tar headers and the disk's own header lie well within it, compressed. +_DISK_START = 1 << 16 +_QCOW2 = b"QFI\xfb" + + +@dataclass(frozen=True) +class DiskLayer: + """What the start of a disk form's layer shows: the first file in it, its owner, and the virtual size of the disk + it holds, a qcow2's from its header, a raw one's its own.""" + + path: str + uid: int + virtual_size: int class UnreadableLayerError(ValueError): @@ -43,6 +61,13 @@ def platform_manifests(ref: str, auth: RegistryAuth | None) -> tuple[str, dict[s def labels(ref: str, arch: str, auth: RegistryAuth | None) -> dict[str, str]: """The labels of ``ref``'s image for ``arch``, read from its config, an index's or a single image's. Raises ValueError when it has none for ``arch``, or as ``read_manifest`` does.""" + return image_config(ref, arch, auth).get("Labels") or {} + + +def image_config(ref: str, arch: str, auth: RegistryAuth | None) -> dict: + """The ``config`` object of ``ref``'s image config for ``arch`` (its environment, entrypoint, command, user, working + folder and labels), an index's or a single image's. Raises ValueError when it has none for ``arch``, or as + ``read_manifest`` does.""" pinned, manifests = platform_manifests(ref, auth) if arch not in manifests: raise ValueError(f"{ref} has no linux/{arch} image") @@ -54,7 +79,28 @@ def labels(ref: str, arch: str, auth: RegistryAuth | None) -> dict[str, str]: config = json.loads(path.read_text()) except OSError as e: raise ValueError(f"its config {manifest['config']['digest']} couldn't be read: {e}") from e - return (config.get("config") or {}).get("Labels") or {} + return config.get("config") or {} + + +def disk_layer(ref: str, digest: str, auth: RegistryAuth | None) -> DiskLayer: + """The disk the image manifest ``digest`` in ``ref``'s repository holds, a disk form's, read from the start of its + one layer. Raises ValueError when it has another number of layers or its start isn't a disk's, + UnreadableLayerError when this Python can't decompress it.""" + layers = read_manifest(f"{ref.partition('@')[0]}@{digest}", auth).get("layers") or [] + if len(layers) != 1: + raise ValueError(f"it has {len(layers)} layers, not the one a disk's image has") + if layers[0].get("mediaType", "").endswith("+zstd"): + raise UnreadableLayerError(f"its layer {layers[0]['digest']} is zstd-compressed, which this reads no part of") + start = read_blob_start(ref, layers[0]["digest"], auth, _DISK_START) + data = zlib.decompressobj(zlib.MAX_WBITS | 32).decompress(start) if start[:2] == b"\x1f\x8b" else start + try: + with tarfile.open(fileobj=io.BytesIO(data), mode="r|") as tar: + member = next(member for member in tar if member.isfile()) + head = tar.extractfile(member).read(32) + except (tarfile.TarError, StopIteration, OSError, EOFError) as e: + raise ValueError(f"the start of its layer {layers[0]['digest']} holds no file: {type(e).__name__}: {e}") from e + size = struct.unpack(">Q", head[24:32])[0] if head[:4] == _QCOW2 and len(head) == 32 else member.size + return DiskLayer(member.name.removeprefix("./"), member.uid, size) def os_release(ref: str, digest: str, auth: RegistryAuth | None) -> dict[str, str]: diff --git a/src/agent_env/store/image_store/registry_api.py b/src/agent_env/store/image_store/registry_api.py index 583716407..61ff38bdb 100644 --- a/src/agent_env/store/image_store/registry_api.py +++ b/src/agent_env/store/image_store/registry_api.py @@ -163,6 +163,26 @@ def read_blob(ref: str, digest: str, auth: RegistryAuth | None, path: Path, *, raise ValueError(f"{ref}: the blob {image.host} served for {digest} hashes to sha256:{hashed.hexdigest()}") +def read_blob_start(ref: str, digest: str, auth: RegistryAuth | None, size: int, *, + timeout: float = _TIMEOUT_SECONDS) -> bytes: + """The first ``size`` bytes of the blob ``digest`` in ``ref``'s repository, or all of a smaller one, read without + fetching the rest, so unchecked against its digest.""" + image = _parse(ref) + if not digest.startswith("sha256:") or not _DIGEST.match(digest): + raise ValueError(f"{ref}: {digest!r} isn't a sha256 digest") + start = bytearray() + with _Repository(image, auth, timeout, ref) as repository: + try: + with repository.stream(f"blobs/{digest}") as response: + for chunk in response.iter_bytes(): + start += chunk + if len(start) >= size: + break + except httpx.HTTPError as e: + raise ValueError(f"{ref}: reading the blob {digest} from {image.host} failed: {type(e).__name__}: {e}") from e + return bytes(start[:size]) + + def write_index(ref: str, images: dict[str, str], auth: RegistryAuth | None, *, timeout: float = _TIMEOUT_SECONDS) -> None: """Write under ``ref``'s tag an index of ``images``, by their platforms, each pinned by its digest in ``ref``'s diff --git a/src/agent_env/store/local_state.py b/src/agent_env/store/local_state.py index d12899439..1fae7751d 100644 --- a/src/agent_env/store/local_state.py +++ b/src/agent_env/store/local_state.py @@ -2,10 +2,11 @@ from __future__ import annotations +import asyncio import hashlib import os -from collections.abc import Callable, Iterable, Iterator -from contextlib import contextmanager +from collections.abc import AsyncIterator, Callable, Iterable, Iterator +from contextlib import asynccontextmanager, contextmanager from pathlib import Path from agent_env.config.errors import ConfigError @@ -18,6 +19,8 @@ fcntl = resource = None _OTHER_FILES = 256 +#: How often a coroutine waiting for a lock another holder has tries it again. +_LOCK_POLL_S = 0.2 def ensure_state_dir(path: Path) -> None: @@ -68,6 +71,27 @@ def holding_locks(ids: Iterable[str], on_wait: Callable[[], None] | None = None) os.close(fd) +@asynccontextmanager +async def awaiting_lock(id: str) -> AsyncIterator[None]: + """``holding_locks`` for one id, waited for on the event loop: tried again every ``_LOCK_POLL_S`` while another + holder has it, so no thread is held waiting, which the holder may need to finish and release it.""" + if fcntl is None: + yield + return + ensure_state_dir(state_root() / "locks") + fd = os.open(_lock_path(id), os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + try: + while True: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + await asyncio.sleep(_LOCK_POLL_S) + yield + finally: + os.close(fd) + + def _allow_open(locks: int) -> None: """Each lock holds a file open, so more locks than the soft limit on open files allows (1024 on Linux, 256 on macOS) raise it toward the hard limit, as Go's runtime does, leaving room for the process's other files.""" diff --git a/tst/integration/computer/disk/rootfs/Dockerfile b/tst/integration/computer/disk/rootfs/Dockerfile new file mode 100644 index 000000000..dba0242df --- /dev/null +++ b/tst/integration/computer/disk/rootfs/Dockerfile @@ -0,0 +1,6 @@ +# A small root filesystem, for either arch, that the disk tests make disks of. +FROM public.ecr.aws/docker/library/busybox:1.37@sha256:a7dbc319b1438115a207e11930dc4bb3e643dcb3ec0bdd694bb36cd8fad289ec +ENV GREETING="hello disk" +USER nobody +WORKDIR /tmp +CMD ["sh", "-c", "echo $GREETING"] diff --git a/tst/integration/computer/disk/tasks/probe.json b/tst/integration/computer/disk/tasks/probe.json new file mode 100644 index 000000000..e1e22b0ab --- /dev/null +++ b/tst/integration/computer/disk/tasks/probe.json @@ -0,0 +1,6 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "@local/agentenv-framework/desktop/desktop", "env_params": {"form": "disk"}}, + {"id": "probe", "type": "env_outcome_verifier", "env_id": "@local/agentenv-framework/desktop/desktop", "file_artifact_id": "@local/agentenv-framework/desktop/check_probe", "verifier_id": "probe", "depends_on": ["env"]}, + {"id": "reset", "type": "reset_env", "env_id": "@local/agentenv-framework/desktop/desktop", "depends_on": ["probe"], "fail_task_on_error": true}, + {"id": "again", "type": "env_outcome_verifier", "env_id": "@local/agentenv-framework/desktop/desktop", "file_artifact_id": "@local/agentenv-framework/desktop/check_probe", "verifier_id": "again", "depends_on": ["reset"]} +] diff --git a/tst/integration/computer/disk_boot_test.py b/tst/integration/computer/disk_boot_test.py new file mode 100644 index 000000000..16fb399b4 --- /dev/null +++ b/tst/integration/computer/disk_boot_test.py @@ -0,0 +1,36 @@ +"""The shipped desktop example's disk form, composed on this machine and booted under KVM in the QEMU launcher through +``agent-env run``: gate 3 probes this machine's arch's disk at full as it composes it, a deploy that names the disk +form boots it with its settings and control token in its seed, it passes the probe at full, and a reset boots it +afresh, where it passes again. Needs Docker, the local registry and /dev/kvm.""" + +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from agent_env.cli import cli +from agent_env.env import Env +from agent_env.providers.sandbox_providers.host_images import host_platform +from agent_env.store.routing import namespace_routing +from tst.integration.computer.conftest import run_context +from tst.util.capabilities import skip_without_kvm + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow, skip_without_kvm()] + +BUNDLE = Path(__file__).parent / "disk" +DESKTOP = "@local/agentenv-framework/desktop/desktop" + + +def test_the_desktops_disk_is_probed_as_composed_then_deployed_probed_and_reset_under_kvm(state): + ran = CliRunner().invoke(cli, ["run", str(BUNDLE), "--task", "probe"]) + + assert ran.exit_code == 0, ran.output + arch = host_platform().rsplit("/", 1)[1] + context = run_context(ran.output) + verifications = context["metadata"]["verifications"] + assert all(result["result"] for step in ("probe", "again") for result in verifications[step]["results"]) + (deployed,) = context["deployed_envs"] + assert (deployed["metadata"]["computer"]["form"], deployed["metadata"]["computer"]["arch"]) == ("disk", arch) + with namespace_routing(): + disk = Env.get(DESKTOP).metadata["composed"]["disk"][arch] + assert (disk["status"], disk["probe"]["passed"], disk["probe"]["level"]) == ("ready", True, "full") diff --git a/tst/integration/computer/disk_test.py b/tst/integration/computer/disk_test.py new file mode 100644 index 000000000..4630d29a0 --- /dev/null +++ b/tst/integration/computer/disk_test.py @@ -0,0 +1,150 @@ +"""A Linux computer's disk form, made by the boot recipe's Dockerfile with the disk tools as a compose builds it: a GPT +disk of an EFI system partition and an ext4 root filesystem that is a container image's, with the image's start in +/etc/agentenv/vm/image.env, in an image holding only /disk/disk.img, owned by uid 107. A disk made over another keeps +its EFI system partition byte for byte and its size, and each arch's disk is made on this machine's own platform, +without emulating the other's. Needs Docker.""" + +import base64 +import io +import json +import subprocess +import tarfile +import uuid +from pathlib import Path + +import pytest + +from agent_env.env.published_images import DISK +from agent_env.providers.sandbox_providers.host_images import host_platform +from agent_env.utils.docker.build import DockerBuildError, build_image + +pytestmark = pytest.mark.integration + +ROOTFS = Path(__file__).parent / "disk" / "rootfs" +UKI = {"amd64": "BOOTX64.EFI", "arm64": "BOOTAA64.EFI"} +ROOT_TYPE = {"amd64": "4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709", "arm64": "B921B045-1DF0-41C3-AF44-4C6F280D3FAE"} +ESP_TYPE = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" +# Each partition of a disk, written out with what the tools read of its filesystems. +_READ = """set -e +cd /work +qemu-img info --output=json disk.img > info.json +qemu-img convert -O raw disk.img disk.raw +sfdisk -J disk.raw > parts.json +python3 -c ' +import json +for i, p in enumerate(json.load(open("parts.json"))["partitiontable"]["partitions"]): + print(p["start"], p["size"], ("esp", "root")[i]) +' | while read -r start size name; do dd if=disk.raw of=$name.img bs=512 skip=$start count=$size status=none; done +mtype -i esp.img ::/EFI/BOOT/$1 > uki +debugfs -R "cat /etc/agentenv/vm/image.env" root.img > image.env 2>/dev/null +debugfs -R "stat /bin/busybox" root.img 2>/dev/null | head -1 > busybox +sha256sum esp.img root.img > sums +rm disk.img disk.raw esp.img root.img +""" + + +@pytest.fixture(scope="module") +def tools(): + tag = f"agentenv-test-disk-tools:{uuid.uuid4().hex[:8]}" + build_image(DISK / "tools" / "Dockerfile", DISK / "tools", tag, platform=None) + yield tag + subprocess.run(["docker", "rmi", "-f", tag], capture_output=True) + + +@pytest.fixture +def images(): + made = [] + yield made + subprocess.run(["docker", "rmi", "-f", *made], capture_output=True) + + +@pytest.mark.parametrize("arch", ["amd64", "arm64"]) +def test_a_disk_is_a_root_filesystem_with_its_start_beside_a_new_esp_and_one_made_over_it_keeps_that_esp( + tools, images, tmp_path, arch): + platform = f"linux/{arch}" + rootfs = _built(images, ROOTFS, platform) + uki = _built(images, _folder(tmp_path / "uki", {"Dockerfile": f"FROM scratch\nCOPY {UKI[arch]} /\n", + UKI[arch]: f"a UKI for {arch}"}), platform) + first = _disk(images, tools, rootfs, uki, platform, tmp_path, SIZE_GB="2") + changed = _built(images, _folder(tmp_path / "changed", {"Dockerfile": f"FROM {rootfs}\nENV EXTRA=1\n"}), platform) + second = _disk(images, tools, changed, first, platform, tmp_path, ESP=first) + + made, over = _read(tools, first, tmp_path / "first", arch), _read(tools, second, tmp_path / "second", arch) + + for disk in (made, over): + assert disk["file"] == ("disk.img", 107, 107) + assert [(p["type"], p["name"]) for p in disk["partitions"]] == [(ESP_TYPE, "ESP"), + (ROOT_TYPE[arch], "agentenv-root")] + assert disk["virtual_size"] == 2 << 30 and disk["uki"] == f"a UKI for {arch}" and disk["busybox"] + assert made["start"] == [("env", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"), + ("env", "GREETING=hello disk"), ("argv", "sh"), ("argv", "-c"), ("argv", "echo $GREETING"), + ("user", "nobody"), ("workdir", "/tmp")] + assert ("env", "EXTRA=1") in over["start"] and over["esp"] == made["esp"] and over["root"] != made["root"] + + +def test_a_new_disk_without_its_size_is_refused_rather_than_made_at_some_default(tools, images, tmp_path): + platform = host_platform() + arch = platform.rsplit("/", 1)[1] + rootfs = _built(images, ROOTFS, platform) + uki = _built(images, _folder(tmp_path / "uki", {"Dockerfile": f"FROM scratch\nCOPY {UKI[arch]} /\n", + UKI[arch]: "a UKI"}), platform) + + with pytest.raises(DockerBuildError, match="a new disk needs its size, SIZE_GB"): + _disk(images, tools, rootfs, uki, platform, tmp_path) + + +def _built(images: list, folder: Path, platform: str) -> str: + tag = f"agentenv-test-disk:{uuid.uuid4().hex[:12]}" + build_image(folder / "Dockerfile", folder, tag, platform=platform) + images.append(tag) + return tag + + +def _disk(images: list, tools: str, rootfs: str, esp: str, platform: str, tmp_path: Path, **args: str) -> str: + """The disk the boot recipe's Dockerfile makes of ``rootfs`` for ``platform``, its ESP from ``esp``, the image a UKI + or a disk is in.""" + config = subprocess.run(["docker", "image", "inspect", "--format", "{{json .Config}}", rootfs], capture_output=True, + text=True, check=True).stdout.strip() + tag = f"agentenv-test-disk:{uuid.uuid4().hex[:12]}" + build_image(DISK / "boot" / "Dockerfile", DISK / "boot", tag, platform=platform, build_args={ + "ROOTFS": rootfs, "TOOLS": tools, "ROOTFS_CONFIG": base64.b64encode(config.encode()).decode(), "ESP": esp, + **args}) + images.append(tag) + return tag + + +def _read(tools: str, image: str, folder: Path, arch: str) -> dict: + """What ``image`` holds: its file with its owner, its disk's virtual size and partitions, the UKI its EFI system + partition holds, the start its root filesystem's image.env gives, and each partition's digest.""" + folder.mkdir() + created = subprocess.run(["docker", "create", image, "none"], capture_output=True, text=True, check=True).stdout.strip() + try: + copied = subprocess.run(["docker", "cp", f"{created}:/disk/disk.img", "-"], capture_output=True, check=True) + finally: + subprocess.run(["docker", "rm", created], capture_output=True) + with tarfile.open(fileobj=io.BytesIO(copied.stdout)) as tar: + (member,) = tar.getmembers() + (folder / "disk.img").write_bytes(tar.extractfile(member).read()) + reader = subprocess.run(["docker", "create", "--platform", host_platform(), "--entrypoint", "sh", tools, "-c", _READ, + "read", UKI[arch]], capture_output=True, text=True, check=True).stdout.strip() + try: + for command in (["cp", f"{folder}/.", f"{reader}:/work"], ["start", "--attach", reader], + ["cp", f"{reader}:/work/.", str(folder)]): + subprocess.run(["docker", *command], check=True, capture_output=True) + finally: + subprocess.run(["docker", "rm", "--force", reader], capture_output=True) + start = [(key, base64.b64decode(value).decode()) for line in (folder / "image.env").read_text().splitlines() + for key, _, value in [line.partition("=")]] + return {"file": (member.name, member.uid, member.gid), "start": start, "uki": (folder / "uki").read_text(), + "virtual_size": json.loads((folder / "info.json").read_text())["virtual-size"], + "partitions": json.loads((folder / "parts.json").read_text())["partitiontable"]["partitions"], + "busybox": (folder / "busybox").read_text().startswith("Inode:"), + **{name.removesuffix(".img"): digest for digest, name in ( + line.split() for line in (folder / "sums").read_text().splitlines())}} + + +def _folder(folder: Path, files: dict[str, str]) -> Path: + folder.mkdir() + for name, text in files.items(): + (folder / name).write_text(text) + return folder diff --git a/tst/unit/artifact/test_computer_artifact.py b/tst/unit/artifact/test_computer_artifact.py index ff257825a..fde1294ac 100644 --- a/tst/unit/artifact/test_computer_artifact.py +++ b/tst/unit/artifact/test_computer_artifact.py @@ -137,7 +137,7 @@ def test_an_image_the_store_has_no_image_artifact_for_refuses_the_put_inspected_ @pytest.mark.parametrize("changes, message", [ ({"os": {"type": "windows", "version": "11"}}, "os.type 'windows' isn't supported yet"), - ({"forms": {"disk": {}}}, "the disk form isn't supported yet"), + ({"forms": {"lume": {}}}, "the lume form isn't supported yet"), ({"arch": ["amd64", "amd64"]}, "lists an architecture twice"), ({"arch": ["riscv64"]}, "arch.0"), ({"session": {"user": "Root User"}}, "session.user"), diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index bef522b39..233f18441 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -107,15 +107,19 @@ def test_a_reused_computer_env_composes_what_it_has_left_on_the_runs_provider_an on_progress=lambda write, line: told.append((write.id, line))) again = materialize(_plan(root), build_here=local_builds(_plan(root)), composing=Composing(provider), on_progress=lambda write, line: told.append((write.id, line))) - env.update_metadata({**env.metadata, "computer": {"id": "c", "version": 1, "images": {"amd64": "a", "arm64": "b"}}, - "composed": {"container": {"amd64": {"status": "ready", "image": "a"}, - "arm64": {"status": "ready", "image": "b"}}}}) + built_ = {form: {arch: {"status": "ready", "image": f"{form}-{arch}"} for arch in ("amd64", "arm64")} + for form in ("container", "disk")} + env.update_metadata({**env.metadata, "composed": built_, "computer": { + "id": "c", "version": 1, "images": {"amd64": "container-amd64", "arm64": "container-arm64"}, + "disks": {"amd64": "disk-amd64", "arm64": "disk-arm64"}}}) built = materialize(_plan(root), build_here=local_builds(_plan(root)), composing=Composing(provider)) assert built.composing == () and checked.composing == again.composing == (env.id,) and all(item.reused for item in again.writes) - assert told == [(env.id, "would compose container/amd64, container/arm64"), (env.id, "composing")] + assert told == [(env.id, "would compose container/amd64, container/arm64, disk/amd64, disk/arm64"), + (env.id, "composing")] assert composed == [(env.id, UBUNTU, REFERENCE, None), - (env.id, UBUNTU, REFERENCE, provider, [("container", "amd64"), ("container", "arm64")])] + (env.id, UBUNTU, REFERENCE, provider, [("container", "amd64"), ("container", "arm64"), + ("disk", "amd64"), ("disk", "arm64")])] def test_a_dry_run_says_what_it_would_compose_of_a_reused_computer_env(tmp_path, composed): @@ -127,7 +131,7 @@ def test_a_dry_run_says_what_it_would_compose_of_a_reused_computer_env(tmp_path, env_id = f"{_plan(root).bundle.bundle.id_root}/desktop" assert dry.materialization.composing == (env_id,) and len(composed) == 1 - assert "envs/desktop: would compose container/amd64, container/arm64" in told + assert "envs/desktop: would compose container/amd64, container/arm64, disk/amd64, disk/arm64" in told def test_the_env_alone_is_planned_whatever_the_bundles_tasks_would_deploy(tmp_path, composed, monkeypatch): diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index 298a90df9..0b8c34fb6 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -29,6 +29,9 @@ OS = {"type": "linux", "distribution": "ubuntu", "version": "24.04", "display": "x11"} +DISK_FORM = {"init": "supervisord", "input_channel": "nocloud", "min_disk_gb": 16} + + CONFIG = {"resolution": {"default": "1280x800", "phase": "deploy"}, "timezone": {"default": "UTC", "phase": "deploy"}, "theme": {"default": "light", "phase": "build"}} @@ -53,7 +56,7 @@ class FakeDocker: def __init__(self, registry: Registry): self.registry, self.builds, self.fail, self.emulates, self.labels = registry, [], {}, {"linux/amd64"}, {} self.probed, self.probe_fails, self.resolved, self.timeouts = [], {}, [], {} - self.writes, self.saved = {}, [] + self.writes, self.saved, self.disk_builds, self.disk_sizes = {}, [], [], {} def resolve(self, request, context, source, arch, work): """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" @@ -81,9 +84,9 @@ def resolving(on, request, source, arch, timeout): return (resolve or self.resolve)(request, on.work / builder.CONTEXT, source, arch, on.work / builder.RESOLVED) return resolving - async def probe(self, links, env, driver, image): + async def probe(self, links, env, driver, image, form="container"): arch = self._arch(self.registry.manifests[image.rpartition("@")[2]]["config"]["digest"]) - self.probed.append((arch, image)) + self.probed.append((arch, image) if form == "container" else (form, arch, image)) if arch in self.probe_fails: checks = [CheckResult(check="P4", name="clicks show their effect", status="failed", detail=self.probe_fails[arch])] else: @@ -96,6 +99,12 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ return if platform in self.fail: raise DockerBuildError(self.fail[platform]) + if build_args is not None or context.name == "tools": + self.disk_builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), + "args": build_args, "context": context.name, "labels": labels}) + args = build_args or {} + self.disk_sizes[tag] = int(args["SIZE_GB"]) if "SIZE_GB" in args else self.disk_sizes.get(args.get("ESP"), 16) + return self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), "files": sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), "conf": next(context.glob("agentenv-*.conf")).read_text(), @@ -122,8 +131,11 @@ def save(self, tag, path): saved_image(path, layers) def push(self, local, ref, store): - self.registry.tags[ref] = self.registry.image(UBUNTU_LAYER, arch=local.rsplit("-", 1)[1], - labels=self.labels.get(local)) + layer = disk_layer(self.disk_sizes.get(local, 16) << 30) if "-disk-" in local else UBUNTU_LAYER + self.registry.tags[ref] = self.registry.image(layer, arch=local.rsplit("-", 1)[1], labels=self.labels.get(local)) + + def image_config(self, tag): + return {"Env": ["PATH=/usr/bin"], "Entrypoint": ["/usr/bin/tini", "--"], "User": "user", "Labels": {"tag": tag}} def joined(self, index, images, auth): self.registry.tags[index] = self.registry.index(**{ @@ -133,6 +145,20 @@ def _arch(self, config_digest): return json.loads(self.registry.blobs[config_digest])["architecture"] +def disk_layer(virtual_size=16 << 30, owner=107, path="disk/disk.img"): + """A disk form's one layer, gzipped: its folder, then a qcow2 of ``virtual_size`` owned by ``owner``.""" + head = b"QFI\xfb" + bytes(20) + virtual_size.to_bytes(8, "big") + bytes(480) + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as tar: + folder = tarfile.TarInfo(path.rpartition("/")[0]) + folder.type = tarfile.DIRTYPE + tar.addfile(folder) + disk = tarfile.TarInfo(path) + disk.size, disk.uid = len(head), owner + tar.addfile(disk, io.BytesIO(head)) + return buffer.getvalue() + + def saved_image(path, layers): """A tar of an image as ``docker save`` writes one: each of ``layers``, bottom first, its history's ``created_by`` (None for a layer with no entry) and its files by path, a file's content as bytes, or with its mode as (bytes, mode), diff --git a/tst/unit/env/envs/conftest.py b/tst/unit/env/envs/conftest.py index 065f6a885..3f561951f 100644 --- a/tst/unit/env/envs/conftest.py +++ b/tst/unit/env/envs/conftest.py @@ -8,13 +8,13 @@ from agent_env.artifact.artifacts import docker_image from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.store import get_artifact_store -from agent_env.env.envs.computer.utils import builder, composer +from agent_env.env.envs.computer.utils import builder, composer, probing from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox from agent_env.store.image_store.image_store import OciRegistryImageStore from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore from agent_env.store.object_store.local.store import LocalFilesystemObjectStore from tst.util.oci_registry import UBUNTU_LAYER, Registry -from tst.unit.env.envs._computer import FakeDocker, _base, _driver +from tst.unit.env.envs._computer import DISK_FORM, FakeDocker, _base, _driver @pytest.fixture @@ -30,12 +30,14 @@ def docker(local_stores, cli_routing, monkeypatch): monkeypatch.setattr(docker_image, "pin_digest", registry.pin_digest) monkeypatch.setattr(builder, "host_platform", lambda: "linux/arm64") monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") - monkeypatch.setattr(composer, "_probe", fake.probe) - monkeypatch.setattr(builder, "_emulated_platforms", lambda: {"linux/arm64", *fake.emulates}) + monkeypatch.setattr(probing, "_probe", fake.probe) + monkeypatch.setattr(builder, "_runs_here", lambda platform, image: platform in {"linux/arm64", *fake.emulates}) monkeypatch.setattr(builder.LocalBuilder, "installed", lambda self, tag, prefix: {"mss": "10.1.0"}) + monkeypatch.setattr(builder.LocalBuilder, "image_config", lambda on, tag: fake.image_config(tag)) monkeypatch.setattr(composer, "_protocol_source", lambda: None) monkeypatch.setattr(builder.LocalBuilder, "resolve", fake.resolving()) monkeypatch.setattr(LocalRegistryImageStore, "ensure_repository", lambda self, repository: None) + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) return fake @@ -48,6 +50,12 @@ def base(docker): return _base(image) +@pytest.fixture +def disk_base(base): + """``base`` with a disk form made from its container when composed.""" + return _base(base.image(), forms={**base.forms.model_dump(exclude_none=True), "disk": DISK_FORM}) + + @pytest.fixture def driver(docker, tmp_path): return _driver(tmp_path / "driver") diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 146480fa3..fd0c4dd57 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -7,11 +7,13 @@ import io import json import logging +import os import re import subprocess import sys import tarfile import time +from pathlib import Path from types import SimpleNamespace import pytest @@ -83,12 +85,94 @@ def test_a_base_built_from_its_context_is_built_here_for_each_arch_and_keyed_by_ "amd64": "unverified", "arm64": "ready"} -def test_the_platforms_this_machine_builds_are_read_from_its_builder(monkeypatch): - inspected = ("Name: desktop-linux\nDriver: docker\nNodes:\nName: desktop-linux\n" - "Platforms: linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64*\nLabels:\n") - monkeypatch.setattr(subprocess, "run", lambda *a, **k: SimpleNamespace(stdout=inspected, returncode=0)) +def test_whether_this_machine_runs_a_platform_is_whether_a_container_of_it_runs_once_per_platform(monkeypatch): + """Not what its builder lists: colima's lists linux/arm64 and linux/386 alone, and runs Rosetta's linux/amd64.""" + ran = [] + monkeypatch.setattr(subprocess, "run", lambda command, **k: ran.append(command) or SimpleNamespace( + returncode=0 if "linux/amd64" in command else 125)) + builder._runs_here.cache_clear() - assert builder._emulated_platforms() == {"linux/arm64", "linux/amd64", "linux/amd64/v2", "linux/riscv64"} + answers = [builder._runs_here(platform, "img") for platform in ("linux/amd64", "linux/riscv64", "linux/amd64")] + builder._runs_here.cache_clear() + + assert answers == [True, False, True] + assert ran == [["docker", "run", "--rm", "--platform", platform, "--entrypoint", "true", "img"] + for platform in ("linux/amd64", "linux/riscv64")] + + +def test_the_platform_check_starts_the_base_or_the_pinned_image_the_shipped_base_builds_from(monkeypatch): + logins = [] + monkeypatch.setattr(builder, "docker_login", logins.append) + registry = SimpleNamespace(image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1@sha256:" + "a" * 64) + context = SimpleNamespace(image_name="local/base-1234:abcd", pinned_name="local/base-1234:abcd") + + assert builder._platform_check(registry) == registry.pinned_name and len(logins) == 1 + assert re.fullmatch(r"public\.ecr\.aws/docker/library/ubuntu:24\.04@sha256:[0-9a-f]{64}", + builder._platform_check(context)) + + +def test_a_resolve_copies_its_inputs_into_the_resolvers_container_and_its_outputs_out_with_no_bind_mount( + tmp_path, monkeypatch): + ran, timeouts = [], [] + + def docker(command, **kwargs): + ran.append(command) + timeouts.append(kwargs.get("timeout")) + if command[1] == "cp" and command[3] == str(tmp_path / "work" / builder.RESOLVED / "out"): + (Path(command[3]) / "result.json").write_text(json.dumps({"apt": [], "wheels": {"driver": []}})) + return subprocess.CompletedProcess(command, 0, "", "") + + work, local = _resolving(tmp_path, monkeypatch, docker) + + result = local.resolve({"apt": [], "python": []}, "base@sha256:" + "b" * 64, "amd64", 60) + + out, container = str(work / builder.RESOLVED / "out"), ran[0][3] + assert result == {"apt": [], "wheels": {"driver": []}} and container.startswith("agentenv-resolve-") + assert ran == [ + ["docker", "create", "--name", container, "--platform", "linux/amd64", "--user", "root", "--entrypoint", + "python3", "base@sha256:" + "b" * 64, "/out/resolve.py", "/out/request.json", "/out"], + ["docker", "cp", f"{work / builder.CONTEXT}/.", f"{container}:/ctx"], + ["docker", "cp", f"{out}/.", f"{container}:/out"], ["docker", "start", "--attach", container], + ["docker", "cp", f"{container}:/out/.", out], ["docker", "rm", "--force", container]] + assert all(0 < timeout <= 60 for timeout in timeouts[:-1]) and timeouts == sorted(timeouts[:-1], reverse=True) + [None] + assert not [arg for command in ran for arg in command if arg in ("-v", "--volume", "--mount")] + assert json.loads((work / builder.RESOLVED / "out" / "request.json").read_text())["owner"] == [os.getuid(), + os.getgid()] + + +@pytest.mark.parametrize("stalls", ["create", "start"]) +def test_one_deadline_bounds_a_resolve_from_its_containers_pull_to_its_output_and_its_container_goes( + tmp_path, monkeypatch, stalls): + """A pull that stalls, as a registry can, ends at the resolve's deadline as one that runs too long does.""" + now, ran = [0.0], [] + + def docker(command, **kwargs): + ran.append(command) + if command[1] == stalls: + now[0] += kwargs["timeout"] + raise subprocess.TimeoutExpired(command, kwargs["timeout"], output="Pulling from acme/base") + now[0] += 0 if command[1] == "rm" else 1 + return subprocess.CompletedProcess(command, 0, "", "") + + monkeypatch.setattr(builder.time, "monotonic", lambda: now[0]) + _, local = _resolving(tmp_path, monkeypatch, docker) + + with pytest.raises(builder.ResolveError, match="resolving took over 60 s") as failed: + local.resolve({"apt": [], "python": []}, "base", "amd64", 60) + + assert now[0] == 60 and ran[-1] == ["docker", "rm", "--force", ran[0][3]] + assert failed.value.log == "Pulling from acme/base" + + +def _resolving(tmp_path, monkeypatch, docker): + """A builder of this machine's Docker in ``tmp_path``, its build context a driver's, ``docker`` its subprocess.run.""" + work = tmp_path / "work" + (work / builder.CONTEXT / "driver").mkdir(parents=True) + (work / builder.CONTEXT / "driver" / "requirements.txt").write_text("mss\n") + monkeypatch.setattr(builder.subprocess, "run", docker) + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + return work, builder.LocalBuilder("LocalSandboxProvider", work, SimpleNamespace( + image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1"), "linux/arm64") def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch): @@ -146,12 +230,13 @@ def test_what_each_run_wrote_is_read_from_the_newest_layer_a_shell_ran_its_comma def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driver, docker, monkeypatch): + """By each builder, and before the check that this machine runs the arch it emulates starts the base.""" signed = [] monkeypatch.setattr(builder, "docker_login", signed.append) _put(base, driver) - assert len(signed) == 2 and len(docker.builds) == 2 + assert len(signed) == 3 and len(docker.builds) == 2 def test_the_builder_vms_helper_lists_plain_files_marks_runnables_links_checks_and_downloads(tmp_path): @@ -185,7 +270,7 @@ def helper(*args): def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_images_are_reused_by_key_here( base, driver, docker, signing, tmp_path, monkeypatch): vms = FakeVmProvider(docker, tmp_path / "vms") - for name in ("build_image", "push_local_image", "docker_login", "_emulated_platforms"): + for name in ("build_image", "push_local_image", "docker_login", "_runs_here"): monkeypatch.setattr(builder, name, lambda *args, **kwargs: pytest.fail("this machine's Docker was used")) told = [] diff --git a/tst/unit/env/envs/test_computer_disk.py b/tst/unit/env/envs/test_computer_disk.py new file mode 100644 index 000000000..ec4e59024 --- /dev/null +++ b/tst/unit/env/envs/test_computer_disk.py @@ -0,0 +1,322 @@ +"""Computer envs of a base with a disk form: each arch's disk is made from its built container, on the base's disk or on +one made from the base's container and the boot recipe, by a build that runs on the builder's own platform; reused by +its key; probed where a provider boots disks natively, else left unverified for a deploy's first boot with KVM to +probe; and written as the computer's disk form. A deploy that names the disk form boots it in a VM, its settings in its +seed, and a reset boots it afresh. Docker and the registry are faked.""" + +import asyncio +import base64 +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from agentenv_protocol import CONTROL_TOKEN_ENV + +from agent_env.artifact.artifacts.computer import ComputerArtifact +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.env import Env +from agent_env.env.envs.computer.utils import boot, composer, probing +from agent_env.env.published_images import DISK +from agent_env.providers.env_providers import env_computer_provider +from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox +from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep +from tst.unit.env.envs._computer import DISK_FORM, _base, _put, disk_layer +from tst.unit.env.envs.test_computer_env import DRIVER_CARD + +BOOT_DOCKERFILE = (DISK / "boot" / "Dockerfile").read_text() + + +def test_each_arch_has_a_disk_made_from_its_container_on_a_base_disk_made_from_the_bases_container(disk_base, driver, + docker): + env = Env.get(_put(disk_base, driver).id) + + containers, disks = env.metadata["composed"]["container"], env.metadata["composed"]["disk"] + assert {arch: r["status"] for arch, r in disks.items()} == {"amd64": "unverified", "arm64": "unverified"} + assert disks["arm64"]["reason"] == "no link of the sandbox provider boots linux/arm64 disks natively to probe it" + assert all(r["image"].startswith("localhost:5000/agentenv-computers:") for r in disks.values()) + built = {build["tag"].split(":")[1].split("-", 1)[1]: build for build in docker.disk_builds + if build["context"] != "tools"} + for arch in ("amd64", "arm64"): + made, disk = built[f"base-disk-{arch}"], built[f"disk-{arch}"] + assert made["dockerfile"] == disk["dockerfile"] == BOOT_DOCKERFILE and made["platform"] == f"linux/{arch}" + assert made["args"]["ROOTFS"] == disk_base.image().image_name and made["args"]["SIZE_GB"] == "16" + assert "ESP" not in made["args"] and made["args"]["TOOLS"] == disk["args"]["TOOLS"] + assert (disk["args"]["ROOTFS"], disk["args"]["ESP"]) == (containers[arch]["image"], made["tag"]) + assert json.loads(base64.b64decode(disk["args"]["ROOTFS_CONFIG"])) == {"Labels": docker.labels[ + f"agentenv-computer:{containers[arch]['key'][7:23]}-{arch}"]} + assert disk["labels"] == {composer.KEY_LABEL: disks[arch]["key"]} + tools = [build for build in docker.disk_builds if build["context"] == "tools"] + assert {(build["platform"], build["tag"]) for build in tools} == {("linux/arm64", built["disk-amd64"]["args"]["TOOLS"]), + ("linux/arm64", built["disk-arm64"]["args"]["TOOLS"])} + assert tools[0]["dockerfile"] == (DISK / "tools" / "Dockerfile").read_text() + computer = env.computer_artifact() + assert computer.forms.disk.model_dump(exclude={"image"}) == {**DISK_FORM, "firmware": ["efi"]} + assert computer.disk_image().image_name.startswith("localhost:5000/agentenv-computers:index-") + assert {arch: r["status"] for arch, r in computer.metadata["validated_images"]["disk"].items()} == { + "amd64": "passed", "arm64": "passed"} + assert env.metadata["computer"]["disks"] == {arch: r["image"] for arch, r in disks.items()} + assert composer.current(env) and composer.pending(env) == [] + + +def test_a_disk_is_reused_by_its_key_of_its_containers_key_the_base_disk_and_the_recipe(disk_base, driver, docker, + monkeypatch): + first = Env.get(_put(disk_base, driver).id).metadata["composed"]["disk"] + made = len(docker.disk_builds) + other = Env.get(_put(disk_base, driver, id="@local/~/kit/other", environment_name="other").id) + after_other = len(docker.disk_builds) + monkeypatch.setattr(composer, "FORMAT", composer.FORMAT + 1) + raised = Env.get(_put(disk_base, driver, id="@local/~/kit/third", environment_name="third").id) + + reused = other.metadata["composed"]["disk"] + assert {arch: (r["key"], r["image"], r["reused"]) for arch, r in reused.items()} == { + arch: (r["key"], r["image"], True) for arch, r in first.items()} and after_other == made + assert all(raised.metadata["composed"]["disk"][arch]["key"] != first[arch]["key"] for arch in first) + assert len(docker.disk_builds) > made + + +def test_a_bases_disk_size_is_in_the_key_and_the_build_of_the_disk_made_for_it(disk_base, driver, docker): + bigger = _base(disk_base.image(), id="@local/~/kit/bigger", forms={ + **disk_base.forms.model_dump(exclude_none=True), "disk": {**DISK_FORM, "min_disk_gb": 32}}) + + small = Env.get(_put(disk_base, driver).id).metadata["composed"]["disk"]["arm64"] + large = Env.get(_put(bigger, driver, id="@local/~/kit/other", environment_name="other").id) + + assert large.metadata["composed"]["disk"]["arm64"]["key"] != small["key"] + assert sorted(build["args"]["SIZE_GB"] for build in docker.disk_builds if "base-disk-arm64" in build["tag"]) == [ + "16", "32"] + + +def test_a_base_with_a_disk_image_or_a_published_one_keeps_its_partition_and_makes_none(disk_base, driver, docker, + monkeypatch): + registry = docker.registry + disks = registry.index(amd64=registry.image(disk_layer()), arm64=registry.image(disk_layer(), arch="arm64")) + image = get_artifact_store().put_document(DockerImageArtifact(id="base-disk", description="", + image_name=f"ghcr.io/acme/base-disk:1@{disks}")) + with_disk = _base(disk_base.image(), id="@local/~/kit/with-disk", forms={ + **disk_base.forms.model_dump(exclude_none=True), "disk": {**DISK_FORM, "image": {"id": image.id, + "version": image.version}}}) + monkeypatch.setattr(composer, "published_disk", lambda sources: "ghcr.io/scaleapi/base-disk:src-1@sha256:" + "a" * 64) + asked = [] + monkeypatch.setattr(LocalRegistryImageStore, "auth", lambda self, ref: asked.append(ref)) + + own = Env.get(_put(with_disk, driver).id).metadata["composed"]["disk"] + published = Env.get(_put(disk_base, driver, id="@local/~/kit/other", environment_name="other").id) + + esps = {build["args"]["ESP"] for build in docker.disk_builds if "ESP" in (build["args"] or {})} + assert esps == {image.image_name, "ghcr.io/scaleapi/base-disk:src-1@sha256:" + "a" * 64} + assert not [build for build in docker.disk_builds if "base-disk" in build["tag"]] + assert {arch: r["status"] for arch, r in own.items()} == {"amd64": "unverified", "arm64": "unverified"} + assert own["arm64"]["key"] != published.metadata["composed"]["disk"]["arm64"]["key"] + assert {image.image_name, "ghcr.io/scaleapi/base-disk:src-1@sha256:" + "a" * 64} <= set(asked) + + +def test_a_disk_isnt_made_for_a_container_that_didnt_build_and_is_made_by_the_next_build(disk_base, driver, docker): + docker.fail["linux/amd64"] = "boom" + env = Env.get(_put(disk_base, driver).id) + disks, computer = env.metadata["composed"]["disk"], env.computer_artifact() + docker.fail.clear() + + composer.compose(env, composer.pending(env)) + + assert (disks["amd64"]["status"], disks["amd64"]["reason"]) == ("skipped", "its container form didn't build (failed)") + assert disks["arm64"]["status"] == "unverified" and computer.arch == ["arm64"] and computer.forms.disk is not None + rebuilt = Env.get(env.id) + assert rebuilt.metadata["composed"]["disk"]["amd64"]["status"] == "unverified" and composer.current(rebuilt) + assert rebuilt.computer_artifact().arch == ["amd64", "arm64"] + + +def test_a_disk_is_probed_on_a_provider_that_boots_its_arch_natively(disk_base, driver, docker, monkeypatch): + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + + env = Env.get(_put(disk_base, driver).id) + + disks = env.metadata["composed"]["disk"] + assert {arch: r["status"] for arch, r in disks.items()} == {"amd64": "unverified", "arm64": "ready"} + assert ("disk", "arm64", disks["arm64"]["image"]) in docker.probed + assert env.computer_artifact().metadata["validated_boot"]["disk"] == {"arm64": disks["arm64"]["probe"]} + + +@pytest.fixture +def vms(monkeypatch): + """The VMs a deploy's boots of a disk ask the local provider for, each serving the desktop driver's card.""" + made = SimpleNamespace(created=[], reset=[]) + + async def create_vm(self, **kwargs): + made.created.append(kwargs) + sandbox = local_sandbox.LocalSandbox(port_map={18765: 20001}, sandbox_id=f"local-vm{len(made.created)}") + sandbox.terminate = AsyncMock() + return sandbox + + async def reset_vm(self, sandbox, **kwargs): + made.reset.append((sandbox.sandbox_id, kwargs)) + + monkeypatch.setattr(local_sandbox.LocalSandboxProvider, "create_vm", create_vm) + monkeypatch.setattr(local_sandbox.LocalSandboxProvider, "reset_vm", reset_vm) + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + monkeypatch.setattr(boot, "ready", AsyncMock(side_effect=lambda url, timeout, token: { + **DRIVER_CARD, "name": "desktop"})) + monkeypatch.setattr(env_computer_provider, "ready", boot.ready) + monkeypatch.setattr(env_computer_provider, "_require_display", AsyncMock()) + return made + + +@pytest.mark.asyncio +async def test_a_deploy_of_the_disk_form_boots_it_in_a_vm_with_its_settings_and_token_in_its_seed(disk_base, driver, + docker, vms): + env = _put(disk_base, driver, display={"width": 1600, "height": 900}) + + deployed = await Env.get(env.id).deploy(env_params={"form": "disk", "computer_settings": {"timezone": "UTC"}}) + + (created,) = vms.created + disk = Env.get(env.id).metadata["composed"]["disk"]["arm64"] + assert (created["image"], created["exposed_ports"], created["cpu"], created["memory"]) == ( + disk["image"], [18765], boot.DISK_CPU, boot.DISK_MEMORY_MB) + assert created["env"]["AGENTENV_SETTING_RESOLUTION"] == "1600x900" and created["env"]["ENVIRONMENT_NAME"] == "desktop" + assert created["env"][CONTROL_TOKEN_ENV] == deployed.control_token + assert (deployed.metadata["computer"]["form"], deployed.metadata["computer"]["image"]) == ("disk", disk["image"]) + assert deployed.environment_url == "http://127.0.0.1:20001" + + +@pytest.mark.asyncio +async def test_an_arch_whose_disk_built_deploys_it_though_the_computer_has_no_disk_form_without_the_other( + disk_base, driver, docker, vms, monkeypatch): + built = composer._built_disk + monkeypatch.setattr(composer, "_built_disk", lambda env, base, arch, *rest: ( + {"status": "failed", "stage": "build", "reason": "boom"} if arch == "amd64" else built(env, base, arch, *rest))) + env = Env.get(_put(disk_base, driver).id) + + deployed = await env.deploy(env_params={"form": "disk"}) + + assert env.computer_artifact().forms.disk is None and env.metadata["composed"]["disk"]["arm64"]["status"] == "ready" + assert deployed.metadata["computer"]["form"] == "disk" and len(vms.created) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fails", [None, "probe", "boot"]) +async def test_first_deploys_of_an_unverified_disk_at_once_share_one_probe_and_its_recorded_result( + disk_base, driver, docker, vms, monkeypatch, fails): + """Whether its probe passes, fails, or never boots it, whose record keeps no image.""" + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) + env = _put(disk_base, driver) + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + probe, first_boot, waited = docker.probe, env_computer_provider.probe_first_boot, [] + + async def slow(*args, **kwargs): + await asyncio.sleep(0.3) + if fails == "boot": + raise RuntimeError("LocalSandboxProvider: no boot") + return await probe(*args, **kwargs) + + async def counted(*args): + waited.append(args[1:3]) + return await first_boot(*args) + + monkeypatch.setattr(probing, "_probe", slow) + monkeypatch.setattr(env_computer_provider, "probe_first_boot", counted) + if fails == "probe": + docker.probe_fails["arm64"] = "the click never landed" + + deployed = await asyncio.gather(*(Env.get(env.id).deploy(env_params={"form": "disk"}) for _ in range(3)), + return_exceptions=True) + + assert waited == [("disk", "arm64")] * 3 + assert len([probed for probed in docker.probed if probed[0] == "disk"]) == (fails != "boot") + recorded = Env.get(env.id).metadata["composed"]["disk"]["arm64"] + if fails: + assert recorded["status"] == "failed" and vms.created == [] + assert recorded["reason"] == ("it didn't boot: LocalSandboxProvider: no boot" if fails == "boot" else + "the probe failed: P4 clicks show their effect: the click never landed") + assert {str(failure) for failure in deployed} == { + f"{env.id}'s disk/arm64 failed its probe on its first boot: {recorded['reason']}"} + else: + assert recorded["status"] == "ready" and len(vms.created) == 3 + assert all(record.metadata["computer"]["form"] == "disk" for record in deployed) + + +@pytest.mark.asyncio +async def test_a_deploy_of_a_disk_bigger_than_the_disk_is_refused_rather_than_booted_smaller(disk_base, driver, docker, + vms): + env = _put(disk_base, driver) + + with pytest.raises(Exception, match="disk is 16 GB, which a deploy doesn't enlarge, and it asks for 64 GB"): + await Env.get(env.id).deploy(disk_size_gb=64, env_params={"form": "disk"}) + + assert vms.created == [] + + +@pytest.mark.asyncio +async def test_an_unverified_disk_is_probed_on_its_first_boot_with_kvm_before_its_deploy_boots_it(disk_base, driver, + docker, vms, + monkeypatch): + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) + env = _put(disk_base, driver) + failing = _put(disk_base, driver, id="@local/~/kit/failing", environment_name="failing") + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + + await Env.get(env.id).deploy(env_params={"form": "disk"}) + docker.probe_fails["arm64"] = "the click never landed" + with pytest.raises(RuntimeError, match="failed its probe on its first boot: the probe failed: P4"): + await Env.get(failing.id).deploy(env_params={"form": "disk"}) + + assert Env.get(env.id).metadata["composed"]["disk"]["arm64"]["status"] == "ready" + assert [probe[0] for probe in docker.probed if probe[0] == "disk"] == ["disk", "disk"] + assert Env.get(failing.id).metadata["composed"]["disk"]["arm64"]["status"] == "failed" + assert len(vms.created) == 1 + + +def test_a_disk_form_is_refused_at_save_where_no_provider_boots_its_disk_natively(disk_base, driver, docker): + env = Env.get(_put(disk_base, driver).id) + + assert env.deploy_refusal(env_params={"form": "disk"}) == ( + f"computer env {env.id!r}: LocalSandboxProvider runs none of its arches (amd64, arm64) natively as disk") + assert env.deploy_refusal() is None + + +@pytest.mark.asyncio +async def test_a_reset_of_a_disk_boots_it_afresh_in_its_vm_with_the_records_settings_and_token(disk_base, driver, + docker, vms, monkeypatch): + env = _put(disk_base, driver, display={"width": 1600, "height": 900}) + deployed = await Env.get(env.id).deploy(env_params={"form": "disk"}) + monkeypatch.setattr(local_sandbox.LocalSandboxProvider, "get_sandbox", AsyncMock( + return_value=local_sandbox.LocalSandbox(port_map={18765: 20001}, sandbox_id="local-vm1"))) + monkeypatch.setattr(env_computer_provider, "end_recordings", AsyncMock()) + + await ResetEnvTaskStep(id="reset", version=None, env_id=env.id, fail_task_on_error=True).execute( + TaskStepContext(deployed_envs=[deployed])) + + ((sandbox_id, kwargs),) = vms.reset + assert (sandbox_id, kwargs["image"]) == ("local-vm1", deployed.metadata["computer"]["image"]) + assert kwargs["env"][CONTROL_TOKEN_ENV] == deployed.control_token + assert kwargs["env"]["AGENTENV_SETTING_RESOLUTION"] == "1600x900" + + +def test_a_computer_artifact_takes_a_disk_form_and_reads_its_image(base, docker): + registry = docker.registry + good = registry.index(amd64=registry.image(disk_layer()), arm64=registry.image(disk_layer(), arch="arm64")) + bad = registry.index(amd64=registry.image(disk_layer(virtual_size=8 << 30, owner=0)), + arm64=registry.image(disk_layer(path="disk/other.img"), arch="arm64")) + images = {name: get_artifact_store().put_document(DockerImageArtifact( + id=name, description="", image_name=f"ghcr.io/acme/{name}:1@{index}")) for name, index in (("good", good), + ("bad", bad))} + + def put(id, disk): + return ComputerArtifact.put(**{**base.model_dump(include={"arch", "os", "session", "config"}), "id": id, + "forms": {**base.forms.model_dump(exclude_none=True), "disk": {**DISK_FORM, **disk}}}) + + derived = put("@local/~/kit/derived", {}) + checked = put("@local/~/kit/good", {"image": {"id": "good", "version": images["good"].version}}) + with pytest.raises(ValueError) as refused: + put("@local/~/kit/bad", {"image": {"id": "bad", "version": images["bad"].version}}) + + assert derived.forms.disk.image is None and derived.disk_image() is None + assert {r["status"] for r in derived.metadata["validated_images"]["disk"].values()} == {"uninspected"} + assert {arch: r["status"] for arch, r in checked.metadata["validated_images"]["disk"].items()} == { + "amd64": "passed", "arm64": "passed"} + assert str(refused.value) == ( + "computer '@local/~/kit/bad': forms.disk (amd64): disk/disk.img is owned by uid 0, not 107; its virtual size is " + "8.0 GiB, under min_disk_gb (16); forms.disk (arm64): its first file is disk/other.img, not disk/disk.img") + assert ComputerArtifact.get(checked.id).forms.disk.image.id == "good" diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 8594ed34d..8a72a38f3 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -37,8 +37,10 @@ from agent_env.env import Env from agent_env.env import env as env_module from agent_env.env.env import DeployedEnv -from agent_env.env.envs.computer.utils import boot, builder, composer, gate, inside, packages, resolve_in_base -from agent_env.env.envs.computer.utils.composer import Composing, _probe +from agent_env.env.envs.computer.utils import (boot, builder, composed, composer, gate, inside, packages, probing, + resolve_in_base) +from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.env.envs.computer.utils.probing import _probe from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer import app_env from agent_env.env.envs.computer.app_env import ComputerAppEnv @@ -248,7 +250,7 @@ async def fake_boot(link, **kwargs): monkeypatch.setattr(boot, "boot", fake_boot) monkeypatch.setattr(boot, "discard", AsyncMock()) - monkeypatch.setattr(composer, "run_probe", AsyncMock(return_value="report")) + monkeypatch.setattr(probing, "run_probe", AsyncMock(return_value="report")) asyncio.run(_probe([SimpleNamespace()], env, driver, "img")) @@ -288,7 +290,7 @@ async def fake_boot(link, **kwargs): monkeypatch.setattr(boot, "boot", fake_boot) monkeypatch.setattr(boot, "discard", AsyncMock()) probed = AsyncMock(return_value="report") - monkeypatch.setattr(composer, "run_probe", probed) + monkeypatch.setattr(probing, "run_probe", probed) assert asyncio.run(_probe([SimpleNamespace()], env, driver, "img")) == ( "report", CARD, "SimpleNamespace") @@ -310,10 +312,10 @@ async def never_ready(*args): async def overran(*args): raise TimeoutError - monkeypatch.setattr(composer, "_probe", never_ready) + monkeypatch.setattr(probing, "_probe", never_ready) unbooted = Env.get(_put(base, driver, id="@local/~/kit/other", environment_name="other").id).metadata[ "composed"]["container"]["arm64"] - monkeypatch.setattr(composer, "_probe", overran) + monkeypatch.setattr(probing, "_probe", overran) stuck = Env.get(_put(base, driver, id="@local/~/kit/stuck", environment_name="stuck").id).metadata[ "composed"]["container"]["arm64"] @@ -324,7 +326,7 @@ async def overran(*args): assert unbooted["reason"].startswith("it didn't boot: LocalSandboxProvider: the computer at http://127.0.0.1:1 " "wasn't ready within 180 s") assert (stuck["status"], stuck["stage"], stuck["reason"]) == ( - "failed", "probe", f"the probe didn't finish within {composer._PROBE_TIMEOUT_S} s") + "failed", "probe", f"the probe didn't finish within {probing._PROBE_TIMEOUT_S} s") @pytest.mark.parametrize("result, stored, replaces", [ @@ -338,7 +340,7 @@ async def overran(*args): ], ids=["unprobed-over-ready", "build-failure-over-built", "building-over-built", "probe-failure-over-unprobed", "probe-failure-over-ready", "ready-over-probe-failure", "building-over-build-failure"]) def test_a_result_takes_a_pairs_place_only_when_it_got_at_least_as_far(result, stored, replaces): - assert (composer._rank(result) >= composer._rank(stored)) is replaces + assert (composed._rank(result) >= composed._rank(stored)) is replaces def test_env_build_builds_what_a_put_left_and_prints_each_pair(base, driver, docker, caplog): diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py new file mode 100644 index 000000000..e3c83e8c4 --- /dev/null +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -0,0 +1,164 @@ +"""The Linux disk form's recipe as the init reads what it writes: the image's start (image-env, run by the disk tools) and +the deploy's environment (the launcher's NoCloud seed), each a line per value base64'd so any bytes survive; and the +local provider booting a disk in the launcher, with KVM and the VM's ports, settings and size.""" + +import asyncio +import base64 +import json +import os +import re +import subprocess +import threading +from unittest.mock import AsyncMock + +import pytest + +from agent_env.env.published_images import DISK +from agent_env.providers.sandbox_providers.local_docker import disk as local_disk +from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider +from agent_env.providers.sandbox_providers.sandbox import SANDBOX_LABEL + +TRICKY = "a value with spaces, = signs, 'quotes', $dollars and\na second line" + + +def _decoded(text: str) -> list[tuple[str, str]]: + """Each line's key and value, as the disk's init decodes them.""" + return [(key, base64.b64decode(value).decode()) for line in text.splitlines() if line and not line.startswith("#") + for key, _, value in [line.partition("=")]] + + +def test_image_env_writes_an_images_start_a_line_a_value_root_and_slash_when_it_names_none(tmp_path): + config = tmp_path / "config.json" + config.write_text(json.dumps({"Env": ["PATH=/usr/bin", f"NOTE={TRICKY}"], "Entrypoint": ["/usr/bin/tini", "--"], + "Cmd": ["start-desktop", "init"], "User": "user", "WorkingDir": "/home/user", + "Labels": {"ignored": "yes"}})) + bare = tmp_path / "bare.json" + bare.write_text(json.dumps({"Cmd": ["sh"]})) + + start, plain = (subprocess.run(["python3", str(DISK / "tools" / "image-env"), str(path)], capture_output=True, + text=True, check=True).stdout for path in (config, bare)) + + assert _decoded(start) == [("env", "PATH=/usr/bin"), ("env", f"NOTE={TRICKY}"), ("argv", "/usr/bin/tini"), + ("argv", "--"), ("argv", "start-desktop"), ("argv", "init"), ("user", "user"), + ("workdir", "/home/user")] + assert _decoded(plain) == [("argv", "sh"), ("user", "root"), ("workdir", "/")] + + +def test_the_seed_holds_each_variable_the_launcher_is_handed_by_its_name_whatever_its_value_holds(tmp_path): + """A value of several lines, a certificate's say, reaches it whole, and one holding a line that looks like another + variable makes none.""" + certificate = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n" + forged = "first line\nVM_ENV_FAKE=1" + + _seed(tmp_path, {"AGENTENV_SETTING_RESOLUTION": "1280x800", "AGENTENV_CONTROL_TOKEN": TRICKY, "EMPTY": "", + "AGENTENV_SETTING_CA": certificate, "AGENTENV_SETTING_NOTE": forged}) + + user_data, meta_data = (tmp_path / "user-data").read_text(), (tmp_path / "meta-data").read_text() + assert user_data.splitlines()[0] == "#agentenv-environment" + assert _decoded(user_data) == [("AGENTENV_CONTROL_TOKEN", TRICKY), ("AGENTENV_SETTING_CA", certificate), + ("AGENTENV_SETTING_NOTE", forged), ("AGENTENV_SETTING_RESOLUTION", "1280x800"), + ("EMPTY", "")] + assert meta_data.endswith("\nlocal-hostname: computer\n") and meta_data.startswith("instance-id: ") + + +def test_the_inits_decoder_keeps_every_byte_of_a_start_and_a_seed_trailing_newlines_included(tmp_path): + """As a container runtime would: $(...) alone drops a value's trailing newlines.""" + decoder = re.search(r"^decode\(\) .*$", (DISK / "boot" / "init").read_text(), re.MULTILINE).group(0) + values = ["plain", "ends with a newline\n", "\n", f"{TRICKY}\n\n"] + config = tmp_path / "config.json" + config.write_text(json.dumps({"Cmd": values, "User": "user\n"})) + start = subprocess.run(["python3", str(DISK / "tools" / "image-env"), str(config)], capture_output=True, text=True, + check=True).stdout + _seed(tmp_path, {f"V{i}": value for i, value in enumerate([*values, "a\nVM_ENV_FAKE=1\n"])}) + seed = (tmp_path / "user-data").read_text() + + def decoded(text: str) -> list[str]: + script = f"{decoder}\n" + "".join(f"decode {line.partition('=')[2]}; printf '%s\\0' \"$decoded\"\n" + for line in text.splitlines() if line and not line.startswith("#")) + return subprocess.run(["sh", "-c", script], capture_output=True, text=True, check=True).stdout.split("\0")[:-1] + + assert decoded(start) == [*values, "user\n", "/"] and decoded(seed) == [*values, "a\nVM_ENV_FAKE=1\n"] + + +def test_the_launcher_gets_kvm_the_vms_ports_and_size_and_its_environment_under_its_prefix(): + args = local_disk.launcher_args([18765, 5900], {"ENVIRONMENT_NAME": "desk", "A": "b c"}, 1.5, 4096) + + assert args == ["--device", "/dev/kvm", "-e", "VM_PORTS=18765 5900", "-e", "VM_CPUS=2", "-e", "VM_MEMORY_MB=4096", + "-e", "VM_ENV_ENVIRONMENT_NAME=ZGVzaw==", "-e", "VM_ENV_A=YiBj"] + + +def _seed(folder, env: dict[str, str]) -> None: + """The seed of ``env`` in ``folder``, as the launcher makes it from what the local provider hands it.""" + args = local_disk.launcher_args([18765], env, 2, 4096) + handed = dict(arg.split("=", 1) for flag, arg in zip(args, args[1:]) if flag == "-e") + subprocess.run(["sh", str(DISK / "launcher" / "seed"), str(folder)], check=True, + env={"PATH": os.environ["PATH"], "HOME": "/root", **handed}) + + +@pytest.fixture +def launched(monkeypatch): + """The ``docker`` commands the local provider runs for a disk, with the launcher image it builds faked.""" + ran = [] + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset({"linux/arm64"})) + monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") + monkeypatch.setattr(local_sandbox, "machine_image", AsyncMock(return_value="agentenv-vm:0123")) + monkeypatch.setattr(local_sandbox, "_free_host_port", lambda: 20001) + monkeypatch.setattr(local_sandbox, "_host_ips", lambda: ("127.0.0.1",)) + monkeypatch.setattr(local_sandbox, "_docker", lambda *args: ran.append(args) or "") + monkeypatch.setattr(LocalSandbox, "pull_images", AsyncMock()) + monkeypatch.setattr(LocalSandbox, "exec_script", AsyncMock(side_effect=lambda script: ran.append(script) or "")) + return ran + + +@pytest.mark.asyncio +async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_reset_boots_it_afresh(launched): + provider = LocalSandboxProvider() + + sandbox = await provider.create_vm(image="localhost:5000/disk@sha256:" + "a" * 64, exposed_ports=[18765], + env={"ENVIRONMENT_NAME": "desk"}, cpu=2, memory=4096) + await provider.reset_vm(sandbox, image="localhost:5000/disk@sha256:" + "a" * 64, env={"ENVIRONMENT_NAME": "again"}) + rebuilt = await provider.get_sandbox(sandbox.sandbox_id) + + first, removed, second = launched + assert first == ("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", + "-p", "127.0.0.1:20001:18765", *local_disk.launcher_args([18765], {"ENVIRONMENT_NAME": "desk"}, + 2, 4096), "agentenv-vm:0123") + assert removed == f"docker rm -f {sandbox.container_name} > /dev/null" + assert second[-2:] == ("VM_ENV_ENVIRONMENT_NAME=YWdhaW4=", "agentenv-vm:0123") and "VM_CPUS=2" in second + LocalSandbox.pull_images.assert_awaited_with(["localhost:5000/disk@sha256:" + "a" * 64]) + assert sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} and rebuilt.owns_container + + +@pytest.mark.asyncio +async def test_a_boot_cancelled_while_docker_starts_the_launcher_waits_for_it_then_tears_it_down(launched, monkeypatch): + started, release, order = threading.Event(), threading.Event(), [] + + def docker(*args): + started.set() + release.wait(5) + order.append("run") + return "" + + monkeypatch.setattr(local_sandbox, "_docker", docker) + monkeypatch.setattr(LocalSandbox, "terminate", AsyncMock(side_effect=lambda: order.append("terminate"))) + booting = asyncio.ensure_future(LocalSandboxProvider().create_vm(image="disk", exposed_ports=[18765], env={})) + await asyncio.to_thread(started.wait, 5) + + booting.cancel() + await asyncio.sleep(0.1) + release.set() + + with pytest.raises(asyncio.CancelledError): + await booting + assert order == ["run", "terminate"] + + +@pytest.mark.asyncio +async def test_a_disk_isnt_booted_where_docker_cant_pass_kvm(launched, monkeypatch): + monkeypatch.setattr(local_sandbox, "kvm_platforms", lambda: frozenset()) + + with pytest.raises(ValueError, match="boots a disk only where this machine's Docker can pass it /dev/kvm"): + await LocalSandboxProvider().create_vm(image="disk", exposed_ports=[18765], env={}) + + assert launched == [] and LocalSandboxProvider.disk_image_platforms() == frozenset() diff --git a/tst/unit/env/published_images_test.py b/tst/unit/env/published_images_test.py index 3973041e3..7dfd67121 100644 --- a/tst/unit/env/published_images_test.py +++ b/tst/unit/env/published_images_test.py @@ -1,21 +1,29 @@ -"""The images agent-env publishes for its infra envs, its agents and an example's env: one per shipped Dockerfile, -tagged by its sources, and the record of their digests a release wheel carries.""" +"""The images agent-env publishes for its infra envs, its agents, an example's env, its base computers and their disk +forms: one per shipped Dockerfile, tagged by its sources (a disk form's by its base's too), and the record of their +digests a release wheel carries.""" +import dataclasses import hashlib import json import shutil +import tomllib import pytest from agent_env.env import bootstrap, published_images from agent_env.env.published_images import ( AGENT_IMAGES, + COMPUTER_IMAGES, + DISK, + DISK_IMAGES, + DISK_TOOLS_IMAGE, EXAMPLE_IMAGES, REGISTRY, PublishedImage, _registry_digest, published_bundle_image, published_digest, + published_disk, release_digests, write_release_digests, ) @@ -26,11 +34,14 @@ def test_every_shipped_dockerfile_is_published_once_under_a_name_of_its_own(): - """The infra envs', every shipped agent's, the env `a2a-agent validate` deploys and every shipped base computer's.""" + """The infra envs', every shipped agent's, the env `a2a-agent validate` deploys, every shipped base computer's, and + the Linux disk form's recipe: its disk tools, its QEMU launcher, and each base's disk, from the boot recipe.""" shipped = [dockerfile for dockerfiles in bootstrap._DOCKERFILES.values() for dockerfile in dockerfiles] shipped += published_images._AGENTS.glob("*/Dockerfile") shipped.append(published_images._EXAMPLES / "mail-mcp" / "envs" / "mail" / "Dockerfile") shipped += published_images._COMPUTERS.glob("*/Dockerfile") + shipped += DISK.glob("*/Dockerfile") + shipped += [DISK / "boot" / "Dockerfile"] * (len(COMPUTER_IMAGES) - 1) assert sorted(image.dockerfile for image in bootstrap.PUBLISHED_IMAGES) == sorted(shipped) assert len({image.name for image in bootstrap.PUBLISHED_IMAGES}) == len(bootstrap.PUBLISHED_IMAGES) @@ -163,3 +174,27 @@ def test_a_shipped_agents_sources_whose_image_isnt_published_find_none(monkeypat assert published_bundle_image(sources) is None + + +def test_a_disk_forms_tag_is_of_its_boot_recipe_its_bases_sources_and_the_disk_tools_and_it_is_found_by_its_bases( + tmp_path, monkeypatch): + (disk,), (base,) = DISK_IMAGES, COMPUTER_IMAGES + recipe = BuildContext.of(disk.dockerfile.parent, disk.dockerfile).source_digest(None).removeprefix("sha256:") + served = {} + monkeypatch.setattr(published_images, "_registry_digest", served.get) + other = tmp_path / "base" + shutil.copytree(base.dockerfile.parent, other) + (other / "start-desktop").write_text("#!/bin/sh\n") + moved = PublishedImage(disk.name, disk.dockerfile, (("ROOTFS", PublishedImage(base.name, other / "Dockerfile")), + ("TOOLS", DISK_TOOLS_IMAGE))) + sources = BuildContext.of(base.dockerfile.parent, base.dockerfile).source_digest(None) + + size = tomllib.loads(base.dockerfile.with_name("artifact.toml").read_text())["forms"]["disk"]["min_disk_gb"] + assert disk.name == "agentenv-ubuntu-24.04-disk" and dict(disk.args) == {"ROOTFS": base, "TOOLS": DISK_TOOLS_IMAGE} + assert disk.build_args == (("SIZE_GB", str(size)),) + assert disk.tag() == "src-" + hashlib.sha256("\0".join( + [recipe, f"ROOTFS={base.tag()}", f"TOOLS={DISK_TOOLS_IMAGE.tag()}", f"SIZE_GB={size}"]).encode()).hexdigest() + assert moved.tag() != disk.tag() != dataclasses.replace(disk, build_args=(("SIZE_GB", "32"),)).tag() + assert published_disk(sources) is None + served[disk.ref()] = _DIGEST + assert published_disk(sources) == f"{disk.ref()}@{_DIGEST}" and published_disk("sha256:" + "0" * 64) is None diff --git a/tst/unit/test_clean_install_wheel.py b/tst/unit/test_clean_install_wheel.py index c1d40778b..44522dbdd 100644 --- a/tst/unit/test_clean_install_wheel.py +++ b/tst/unit/test_clean_install_wheel.py @@ -1,5 +1,6 @@ """The wheel check in ``.github/scripts/clean_install.py``: every tracked file of each shipped bundle, and every bundle -entry point, must be in the built wheel.""" +entry point, must be in the built wheel; and the published images as the checkout makes them, which +``check_clean_install.py`` compares the installed ones' tags with.""" from __future__ import annotations @@ -10,6 +11,10 @@ import pytest +import agent_env +from agent_env.env import bootstrap +from tst.util.scripts import load_script + SCRIPT = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "clean_install.py" sys.path.insert(0, str(SCRIPT.parent)) # as running the script does, so it imports its sibling shipped_bundles _spec = importlib.util.spec_from_file_location("clean_install", SCRIPT) @@ -69,3 +74,13 @@ def test_the_gate_checks_the_folder_of_every_shipped_bundle(): "echo-mcp": "src/agent_env/examples/echo-mcp", "hello": "src/agent_env/examples/hello", "mail-mcp": "src/agent_env/examples/mail-mcp", "next-env-event": "src/agent_env/examples/next-env-event"} + + +def test_the_checkout_makes_each_published_image_as_the_installed_sources_do_when_they_are_its_own(monkeypatch): + """Every field of an image is kept in the checkout's copy, its build args too, so the tags differ only when the + sources do.""" + check = load_script("check_clean_install.py") + monkeypatch.setattr(check, "CHECKOUT_SRC", Path(agent_env.__file__).parent.parent) + + assert [check._checkout(image) for image in bootstrap.PUBLISHED_IMAGES] == list(bootstrap.PUBLISHED_IMAGES) + assert any(image.build_args for image in bootstrap.PUBLISHED_IMAGES) diff --git a/tst/unit/test_infra_build_script.py b/tst/unit/test_infra_build_script.py index dfb2c44c6..da1246a5c 100644 --- a/tst/unit/test_infra_build_script.py +++ b/tst/unit/test_infra_build_script.py @@ -4,6 +4,7 @@ from __future__ import annotations import ast +import base64 import json import subprocess import sys @@ -12,7 +13,7 @@ import pytest from agent_env.env import bootstrap -from agent_env.env.published_images import REGISTRY, write_release_digests +from agent_env.env.published_images import DISK_IMAGES, REGISTRY, write_release_digests from tst.util.scripts import SCRIPTS, load_script REPO = SCRIPTS.parents[1] @@ -26,6 +27,18 @@ def script(): return load_script(SCRIPT.name) +@pytest.fixture +def workflow(monkeypatch, tmp_path) -> Path: + """What a workflow step's environment holds: its output file, which is returned, the actor and the job's token, the + repository and the commit.""" + output = tmp_path / "output" + output.touch() + for name, value in {"GITHUB_OUTPUT": str(output), "GITHUB_ACTOR": "bot", "GHCR_TOKEN": "t0ken", + "GITHUB_REPOSITORY": "scaleapi/agentenv-framework", "GITHUB_SHA": "c0ffee"}.items(): + monkeypatch.setenv(name, value) + return output + + def _run(script, monkeypatch, *args: str) -> int: monkeypatch.setattr(sys, "argv", [SCRIPT.name, *args]) return script.main() @@ -46,7 +59,8 @@ def test_it_imports_only_the_standard_library(): def test_its_images_are_the_ones_agent_env_ships(script): assert script.REGISTRY == REGISTRY assert script.IMAGES == {image.name: str(image.dockerfile.relative_to(REPO)) for image in bootstrap.PUBLISHED_IMAGES} - assert set(script._SMOKES) | {"agentenv-service-db"} == set(script.IMAGES) + assert set(script._SMOKES) | {"agentenv-service-db", *script.DISK_ARGS} == set(script.IMAGES) + assert script.DISK_ARGS == {image.name: {arg: built.name for arg, built in image.args} for image in DISK_IMAGES} def test_check_accepts_every_image_the_plan_lists(script, monkeypatch, capsys): @@ -62,8 +76,22 @@ def _gateway(**changes) -> str: return json.dumps({key: value for key, value in {**entry, **changes}.items() if value is not None}) +def _disk(**args) -> str: + image = "agentenv-ubuntu-24.04-disk" + return json.dumps({"name": image, "repository": f"{REGISTRY}/{image}", "tag": _TAG, + "dockerfile": "src/agent_env/computers/disk/linux/boot/Dockerfile", + "context": "src/agent_env/computers/disk/linux/boot", "args": { + "ROOTFS": f"{REGISTRY}/agentenv-ubuntu-24.04:{_TAG}", "TOOLS": f"{REGISTRY}/agentenv-disk-tools:{_TAG}", + "SIZE_GB": "16", **args}}) + + @pytest.mark.parametrize("entry, problem", [ (_gateway(name="agentenv-evil"), "'agentenv-evil' isn't an image agent-env ships"), + (_gateway(args={"ROOTFS": "evil"}), "agentenv-gateway: unexpected args"), + (_disk(ROOTFS="ghcr.io/someone/ubuntu:latest"), "agentenv-ubuntu-24.04-disk: ROOTFS is 'ghcr.io/someone/ubuntu:latest'"), + (_disk(TOOLS=f"{REGISTRY}/agentenv-disk-tools:latest"), "agentenv-ubuntu-24.04-disk: TOOLS is 'ghcr.io/scaleapi/"), + (_disk(EXTRA="x"), "agentenv-ubuntu-24.04-disk: args is "), + (_disk(SIZE_GB="32"), "agentenv-ubuntu-24.04-disk: SIZE_GB is '32', not its base's min_disk_gb, 16"), (_gateway(repository="ghcr.io/someone/agentenv-gateway"), "agentenv-gateway: repository is 'ghcr.io/someone/"), (_gateway(dockerfile="tst/data/agentenv_mcp/Dockerfile"), "agentenv-gateway: dockerfile is 'tst/data/agentenv_mcp"), (_gateway(context="."), "agentenv-gateway: context is '.'"), @@ -109,6 +137,100 @@ def test_exists_says_yes_no_or_that_it_cant_tell(script, monkeypatch, capsys, an assert ("::error::couldn't tell whether the registry has" in capsys.readouterr().out) is (code == 2) +@pytest.mark.parametrize("answer, code, output", [(_DIGEST, 0, "build=false\n"), ("ERROR: not found", 0, "build=true\n"), + ("ERROR: connection refused", 1, "")], ids=["served", "missing", "unknown"]) +def test_wanted_signs_in_and_builds_only_a_tag_the_registry_says_it_lacks(script, monkeypatch, capsys, workflow, answer, + code, output): + calls, tokens = [], [] + docker = _docker({f"{REGISTRY}/agentenv-gateway:{_TAG}": answer}, calls) + + def run(command, **kw): + tokens.append(kw.get("input")) + return docker(command, **kw) + + monkeypatch.setattr(script.subprocess, "run", run) + + assert _run(script, monkeypatch, "wanted", _gateway()) == code + + assert (calls[0], tokens[0]) == (["docker", "login", "ghcr.io", "--username", "bot", "--password-stdin"], "t0ken") + assert workflow.read_text() == output + assert ("is published already, so it isn't built again" in capsys.readouterr().out) is (answer == _DIGEST) + + +@pytest.mark.parametrize("command", [["wanted"], ["build", "amd64"]]) +def test_wanted_and_build_refuse_an_entry_before_signing_in_or_building(script, monkeypatch, capsys, workflow, command): + calls, runs = [], [] + monkeypatch.setattr(script.subprocess, "run", _docker({}, calls)) + monkeypatch.setattr(script.subprocess, "Popen", lambda argv, **kw: runs.append(argv)) + + assert _run(script, monkeypatch, command[0], _gateway(context="."), *command[1:]) == 1 + + assert calls == runs == [] and workflow.read_text() == "" + assert "::error::agentenv-gateway: context is '.'" in capsys.readouterr().out + + +def test_build_builds_an_image_for_its_platform_under_its_tag_with_its_source_on_it(script, monkeypatch, workflow): + runs, waits = _attempts(script, monkeypatch, [(_LIMITED, 1), ("#1 DONE\n", 0)]) + + assert _run(script, monkeypatch, "build", _gateway(), "arm64") == 0 + + assert runs[1] == ["docker", "build", "--platform", "linux/arm64", "--provenance=false", "-f", + "src/agent_env/env/gateway/Dockerfile", "-t", f"{REGISTRY}/agentenv-gateway:{_TAG}-arm64", + "--label", "org.opencontainers.image.source=https://github.com/scaleapi/agentenv-framework", + "--label", "org.opencontainers.image.revision=c0ffee", "--label", f"io.agentenv.source-tag={_TAG}", + "src/agent_env/env/gateway"] + assert runs[0] == runs[1] and waits == [script._RETRY_WAITS_SECONDS[0]] + + +def test_build_makes_a_disk_form_of_its_bases_image_of_the_arch_with_that_images_config_the_tools_and_its_size( + script, monkeypatch, workflow): + runs, _ = _attempts(script, monkeypatch, [("pulled\n", 0), ("#1 DONE\n", 0)]) + inspected = [] + config = '{"Env": ["A=1"]}' + monkeypatch.setattr(script.subprocess, "run", lambda command, **kw: inspected.append(command) or ( + subprocess.CompletedProcess(command, 0, config + "\n", ""))) + rootfs = f"{REGISTRY}/agentenv-ubuntu-24.04:{_TAG}-amd64" + + assert _run(script, monkeypatch, "build", _disk(), "amd64") == 0 + + assert runs[0] == ["docker", "pull", "--platform", "linux/amd64", rootfs] + assert inspected == [["docker", "image", "inspect", "--format", "{{json .Config}}", rootfs]] + assert runs[1][runs[1].index("-t"):runs[1].index("--label")] == [ + "-t", f"{REGISTRY}/agentenv-ubuntu-24.04-disk:{_TAG}-amd64", "--build-arg", f"ROOTFS={rootfs}", + "--build-arg", f"TOOLS={REGISTRY}/agentenv-disk-tools:{_TAG}", "--build-arg", "SIZE_GB=16", + "--build-arg", f"ROOTFS_CONFIG={base64.b64encode(config.encode()).decode()}"] + + +def test_publish_tags_each_new_images_platform_builds_as_one_and_leaves_a_published_tag_as_it_is(script, monkeypatch, + capsys, workflow): + published, new = f"{REGISTRY}/agentenv-gateway:{_TAG}", f"{REGISTRY}/agentenv-ubuntu-24.04-disk:{_TAG}" + calls = [] + monkeypatch.setattr(script.subprocess, "run", _docker({published: _DIGEST}, calls)) + runs, _ = _attempts(script, monkeypatch, [("pushed\n", 0)]) + + assert _run(script, monkeypatch, "publish", json.dumps([json.loads(_gateway()), json.loads(_disk())])) == 0 + + assert calls[0] == ["docker", "login", "ghcr.io", "--username", "bot", "--password-stdin"] + assert runs == [["docker", "buildx", "imagetools", "create", "-t", new, + "--annotation", "index:org.opencontainers.image.source=https://github.com/scaleapi/agentenv-framework", + "--annotation", "index:org.opencontainers.image.revision=c0ffee", f"{new}-amd64", f"{new}-arm64"]] + assert f"{published} is published already, so it's left as it is" in capsys.readouterr().out + + +@pytest.mark.parametrize("first, served", [ + (_gateway(context="."), {}), + (_gateway(), {f"{REGISTRY}/agentenv-gateway:{_TAG}": "ERROR: connection refused"}), +], ids=["refused", "unknown"]) +def test_publish_tags_nothing_more_after_an_entry_it_refuses_or_whose_tag_it_cant_look_up(script, monkeypatch, workflow, + first, served): + monkeypatch.setattr(script.subprocess, "run", _docker(served)) + runs, _ = _attempts(script, monkeypatch, []) + + assert _run(script, monkeypatch, "publish", json.dumps([json.loads(first), json.loads(_disk())])) == 1 + + assert runs == [] + + def _record(tmp_path, digests: dict[str, str] | None = None) -> Path: path = tmp_path / "published_images.json" write_release_digests(digests or {image.ref(): _DIGEST for image in bootstrap.PUBLISHED_IMAGES}, path) @@ -197,6 +319,58 @@ def test_the_service_db_smoke_waits_for_its_health_check_and_removes_the_contain assert calls[0] == ["docker", "run", "-d", "pg"] and calls[-1] == ["docker", "rm", "-f", "c123"] +@pytest.mark.parametrize("kvm, machine, how", [(True, "aarch64", "with KVM"), (False, "aarch64", "under TCG"), + (True, "x86_64", "under TCG")], ids=["kvm", "no-kvm", "another-arch"]) +def test_a_disk_forms_smoke_boots_it_in_the_launcher_until_its_desktop_runs(script, monkeypatch, capsys, kvm, machine, + how): + calls, logs = [], iter(["agentenv-init: starting", "success: xvfb entered RUNNING\nsuccess: dbus entered RUNNING\n" + "success: desktop entered RUNNING"]) + + def run(command, **kw): + calls.append((command, kw.get("input"))) + answers = {"inspect": "arm64\n", "run": "c1\n", "logs": next(logs) if command[1] == "logs" else ""} + return subprocess.CompletedProcess(command, 0, answers.get(command[1], answers.get(command[2], "")), "") + + monkeypatch.setattr(script.subprocess, "run", run) + monkeypatch.setattr(script.time, "sleep", lambda seconds: None) + monkeypatch.setattr(script.Path, "exists", lambda self: kvm) + monkeypatch.setattr(script.platform, "machine", lambda: machine) + + assert _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1-arm64", "--launcher", "launcher:1") == 0 + + commands = [command for command, _ in calls] + assert calls[1] == (["docker", "build", "-q", "-t", "disk:1-arm64-machine", "-"], + "FROM --platform=linux/arm64 disk:1-arm64 AS disk\nFROM launcher:1\nCOPY --from=disk /disk/ /disk/\n") + assert commands[2] == ["docker", "run", "-d", *(["--device", "/dev/kvm"] if kvm else []), "-e", "VM_ARCH=arm64", + "disk:1-arm64-machine"] + assert commands[-1] == ["docker", "rm", "-f", "c1"] and commands.count(["docker", "logs", "c1"]) == 2 + assert f"booted to its desktop {how}" in capsys.readouterr().out + + +def test_a_disk_forms_smoke_waits_for_every_program_its_bases_supervisord_conf_declares(script, monkeypatch, capsys, + tmp_path): + conf = tmp_path / Path(script.IMAGES["agentenv-ubuntu-24.04"]).with_name("supervisord.conf") + conf.parent.mkdir(parents=True) + conf.write_text("[supervisord]\nnodaemon=true\n\n[program:xvfb]\ncommand=Xvfb\n\n[program:recorder]\ncommand=rec\n") + monkeypatch.setattr(script, "REPO", tmp_path) + monkeypatch.setattr(script, "_BOOTED_WITHIN_SECONDS", 0.01) + monkeypatch.setattr(script.time, "sleep", lambda seconds: None) + console = "success: xvfb entered RUNNING\nsuccess: dbus entered RUNNING\nsuccess: desktop entered RUNNING\n" + monkeypatch.setattr(script.subprocess, "run", lambda command, **kw: subprocess.CompletedProcess( + command, 0, {"image": "amd64\n", "run": "c1\n", "logs": console}.get(command[1], ""), "")) + + assert _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1", "--launcher", "launcher:1") == 1 + + assert "::error::disk:1 didn't boot to its desktop within 0.01s" in capsys.readouterr().out + + +def test_a_disk_forms_smoke_needs_the_launcher_to_boot_it_in(script, monkeypatch, capsys): + with pytest.raises(SystemExit) as exited: + _run(script, monkeypatch, "smoke", "agentenv-ubuntu-24.04-disk", "disk:1") + + assert exited.value.code == 2 and "--launcher" in capsys.readouterr().err + + def test_a_service_db_that_never_turns_healthy_fails_with_its_logs(script, monkeypatch, capsys): calls = [] monkeypatch.setattr(script.subprocess, "run", _docker({}, calls, [])) diff --git a/tst/unit/test_infra_images_script.py b/tst/unit/test_infra_images_script.py index 5e9921857..71be6dd6a 100644 --- a/tst/unit/test_infra_images_script.py +++ b/tst/unit/test_infra_images_script.py @@ -4,6 +4,7 @@ import json import sys +import tomllib import pytest @@ -50,19 +51,29 @@ def test_plan_lists_every_image_with_its_checkout_paths_and_tag(script, monkeypa "context": "src/agent_env/env/gateway"} -def test_plan_only_lists_the_infra_images_or_the_agents_images(script, monkeypatch, capsys): - """The release builds the infra images before it publishes the protocol, and the images that install it, the - bundles' and the Playwright MCP browser's, after.""" +def test_plan_only_lists_the_infra_images_the_agents_images_or_the_disk_forms(script, monkeypatch, capsys): + """The release builds the infra images before it publishes the protocol, the images that install it, the bundles' + and the Playwright MCP browser's, after, and the disk forms over their base computers' published images.""" planned = {} - for only in ("infra", "agents"): + for only in ("infra", "agents", "disks"): assert _run(script, monkeypatch, "plan", "--only", only) == 0 - planned[only] = [entry["name"] for entry in json.loads(capsys.readouterr().out)] + planned[only] = json.loads(capsys.readouterr().out) bundles = [image.name for image in BUNDLE_IMAGES] after = [*bundles, "agentenv-playwright-mcp"] # the browser's recorder installs the protocol too - assert planned == {"infra": [image.name for image in bootstrap.PUBLISHED_IMAGES if image.name not in after], - "agents": after} - assert "agentenv-gateway" in planned["infra"] and bundles == [ + disks = ["agentenv-ubuntu-24.04-disk"] + names = {only: [entry["name"] for entry in entries] for only, entries in planned.items()} + assert names == {"infra": [image.name for image in bootstrap.PUBLISHED_IMAGES if image.name not in after + disks], + "agents": after, "disks": disks} + assert "agentenv-disk-tools" in names["infra"] and "agentenv-qemu-launcher" in names["infra"] + (disk,) = planned["disks"] + base = next(image for image in BUNDLE_IMAGES if image.name == "agentenv-ubuntu-24.04") + assert disk["args"] == {"ROOTFS": base.ref(), "TOOLS": next(image.ref() for image in bootstrap.PUBLISHED_IMAGES + if image.name == "agentenv-disk-tools"), + "SIZE_GB": str(tomllib.loads(base.dockerfile.with_name("artifact.toml").read_text())[ + "forms"]["disk"]["min_disk_gb"])} + assert "args" not in planned["infra"][0] + assert "agentenv-gateway" in names["infra"] and bundles == [ "agentenv-claude-code", "agentenv-codex", "agentenv-gemini", "agentenv-hermes", "agentenv-letta-code", "agentenv-opencode", "agentenv-openhands", "agentenv-pi", "agentenv-proxy", "agentenv-pydantic-ai", "agentenv-mail-mcp", "agentenv-ubuntu-24.04"] diff --git a/tst/unit/test_release_infra_images.py b/tst/unit/test_release_infra_images.py index f143651ae..db7f189a2 100644 --- a/tst/unit/test_release_infra_images.py +++ b/tst/unit/test_release_infra_images.py @@ -21,9 +21,11 @@ # What installs or runs agent-env and its dependencies in a job. _RUNS_DEPENDENCIES = re.compile(r"\buv (sync|run)\b|infra_images\.py") # What reads the shipped bundles' folders: an image's tag, its build, and the wheel. -_READS_SHIPPED_SOURCES = re.compile(r"infra_images\.py (plan|write)|docker build\b|uv build --no-sources --out-dir dist") +_READS_SHIPPED_SOURCES = re.compile( + r"infra_images\.py (plan|write)|infra_build\.py build\b|docker build\b|uv build --no-sources --out-dir dist") # The pin command, and not pin-pyproject, which pins the framework's own requirement. _PIN = re.compile(r"python3 \.github/scripts/shipped_bundles\.py pin(?![\w-])") +_SCRIPT = "python3 .github/scripts/infra_build.py" def _runs(job: str) -> list[str]: @@ -56,36 +58,46 @@ def test_a_job_that_installs_agent_env_holds_no_token_that_writes(job): def test_the_jobs_that_push_or_publish_check_what_they_are_handed(): - build = "\n".join(_runs("infra-build")) - publish = "\n".join(_runs("infra-publish")) - - assert 'infra_build.py check "$ENTRY"' in build and 'infra_build.py check "$entry"' in publish - assert build.index("infra_build.py check") < build.index("docker build") - assert publish.index("infra_build.py check") < publish.index("imagetools create") + """The script's wanted, build and publish check each entry before anything else (``test_infra_build_script``).""" + assert _step("infra-build", "wanted")["run"] == f'{_SCRIPT} wanted "$ENTRY"' + assert f'{_SCRIPT} build "$ENTRY" "$ARCH"' in _runs("infra-build") + assert _runs("infra-publish")[-1] == f'{_SCRIPT} publish "$MISSING"' assert "infra_build.py verify-record " + RECORD in _step("framework", "record")["run"] -def test_a_published_tag_is_never_built_again_or_moved(): - wanted = _step("infra-build", "wanted")["run"] - publish = "\n".join(_runs("infra-publish")) +@pytest.mark.parametrize("job", ["infra-build", "agent-build", "disk-build"]) +def test_a_published_tag_is_never_built_again(job): + """Nor moved: publish leaves a tag the registry serves as it is (``test_infra_build_script``).""" + steps = JOBS[job]["steps"] - assert 'infra_build.py exists "$REF"' in wanted assert all(step.get("if") == "steps.wanted.outputs.build == 'true'" - for step in JOBS["infra-build"]["steps"] if step.get("name") in ("Build", "Check that it runs", - "Push it under its tag and platform")) - assert publish.index('infra_build.py exists "$ref"') < publish.index("imagetools create") + for step in steps[steps.index(_step(job, "wanted")) + 1:]) + + +@pytest.mark.parametrize(("job", "infra_job", "listed"), [("agent-build", "infra-build", "missing_agents"), + ("agent-publish", "infra-publish", "missing_agents"), + ("disk-publish", "infra-publish", "missing_disks")]) +def test_the_agent_and_disk_image_jobs_are_the_infra_image_jobs_on_their_lists(job, infra_job, listed): + """So each check here on how the infra images are built, pushed and tagged holds for the agents' images and the + disk forms too.""" + def body(name: str) -> dict: + return {key: value for key, value in JOBS[name].items() if key not in ("needs", "if")} + assert yaml.safe_load(yaml.safe_dump(body(infra_job)).replace("missing_infra", listed)) == body(job) -@pytest.mark.parametrize(("agent_job", "infra_job"), [("agent-build", "infra-build"), ("agent-publish", "infra-publish")]) -def test_the_agent_image_jobs_are_the_infra_image_jobs_on_the_agents_list(agent_job, infra_job): - """So each check here on how the infra images are built, pushed and tagged holds for the agents' images too.""" - def body(job: str) -> dict: - return {key: value for key, value in JOBS[job].items() if key not in ("needs", "if")} - assert yaml.safe_load(yaml.safe_dump(body(infra_job)).replace("missing_infra", "missing_agents")) == body(agent_job) +def test_the_disk_build_is_the_infra_build_but_for_the_launcher_it_pulls_builds_and_boots_the_disk_in(): + infra, disk = JOBS["infra-build"], JOBS["disk-build"] + own = re.compile(r"mirror_base_images\.py|infra_build\.py smoke") + assert {key: disk[key] for key in ("runs-on", "timeout-minutes", "permissions", "env")} == { + key: infra[key] for key in ("runs-on", "timeout-minutes", "permissions", "env")} + assert [step for step in disk["steps"] if step in infra["steps"]] == [ + step for step in infra["steps"] if not own.search(step.get("run", ""))] -@pytest.mark.parametrize("job", ["infra-build", "infra-publish", "agent-build", "agent-publish"]) + +@pytest.mark.parametrize("job", ["infra-build", "infra-publish", "agent-build", "agent-publish", "disk-build", + "disk-publish"]) def test_nothing_is_pushed_before_mains_ci_passes(job): """A condition with ``!cancelled()`` runs whatever the jobs it needs ended as, so it checks main-ci's itself.""" assert "main-ci" in JOBS[job]["needs"] @@ -113,7 +125,7 @@ def test_the_framework_waits_for_every_image_job(): assert _condition("framework").startswith("!cancelled()") assert ("(needs.infra-plan.result == 'skipped' || (needs.infra-plan.result == 'success' " "&& needs.infra-record.result == 'success'))") in _condition("framework") - for image_job in ("infra-build", "infra-publish", "agent-build", "agent-publish"): + for image_job in ("infra-build", "infra-publish", "agent-build", "agent-publish", "disk-build", "disk-publish"): assert image_job in JOBS["infra-record"]["needs"] and _finished(image_job) in _condition("infra-record") @@ -121,7 +133,7 @@ def test_every_job_that_reads_the_shipped_bundles_pins_the_protocol_first(): """The pin is a source of the shipped envs: a job without it would tag, build, record or ship other sources.""" readers = {job: "\n".join(_runs(job)) for job in JOBS if _READS_SHIPPED_SOURCES.search("\n".join(_runs(job)))} - assert set(readers) == {"infra-plan", "infra-build", "agent-build", "infra-record", "framework"} + assert set(readers) == {"infra-plan", "infra-build", "agent-build", "disk-build", "infra-record", "framework"} for job, runs in readers.items(): pins = [pin.start() for pin in _PIN.finditer(runs)] assert len(pins) == 1 and pins[0] < _READS_SHIPPED_SOURCES.search(runs).start(), job @@ -186,23 +198,22 @@ def test_a_build_pulls_its_base_images_from_the_mirror_first(): assert step["run"] == 'python3 .github/scripts/mirror_base_images.py "$DOCKERFILE"' assert step["if"] == "steps.wanted.outputs.build == 'true'" - assert build.index("mirror_base_images.py") < build.index("docker build") + assert build.index("mirror_base_images.py") < build.index("infra_build.py build") def test_what_a_registry_can_rate_limit_is_tried_again(): - """A 429 on a base image's pull would fail the build, and with it the release's PyPI jobs.""" + """A 429 on a base image's pull would fail the build, and with it the release's PyPI jobs; the script's build and + publish try again themselves (``test_infra_build_script``).""" build = "\n".join(_runs("infra-build")) - publish = "\n".join(_runs("infra-publish")) - assert re.search(r"infra_build\.py retry \\\s+docker build", build) assert 'infra_build.py retry docker push "$BUILT"' in build - assert "infra_build.py retry docker buildx imagetools create" in publish assert JOBS["infra-build"]["strategy"]["max-parallel"] <= 4 def test_every_infra_image_pins_its_bases_by_digest_and_dependabot_bumps_them(): """A release rebuilds an image only when its sources change, so a base behind a bare tag never picks up its - fixes; pinned, the pin is a source, and Dependabot bumping it is what rebuilds and republishes the image.""" + fixes; pinned, the pin is a source, and Dependabot bumping it is what rebuilds and republishes the image. A base a + build arg names is another published image, whose tag is a source of this one's.""" dependabot = yaml.safe_load((REPO / ".github" / "dependabot.yml").read_text()) (docker,) = [update for update in dependabot["updates"] if update["package-ecosystem"] == "docker"] @@ -210,7 +221,7 @@ def test_every_infra_image_pins_its_bases_by_digest_and_dependabot_bumps_them(): text = image.dockerfile.read_text() stages = set(re.findall(r"^FROM\s+\S+\s+AS\s+(\S+)", text, re.IGNORECASE | re.MULTILINE)) bases = [ref for ref in re.findall(r"^FROM\s+(?:--\S+\s+)*(\S+)", text, re.IGNORECASE | re.MULTILINE) - if ref not in stages] + if ref not in stages and ref != "scratch" and not ref.startswith("${")] assert bases and all(re.search(r"@sha256:[0-9a-f]{64}$", ref) for ref in bases), (image.name, bases) assert f"/{image.dockerfile.parent.relative_to(REPO)}" in docker["directories"], image.name @@ -223,3 +234,15 @@ def test_dependabot_proposes_a_bases_fixes_and_never_an_upgrade(): assert {"dependency-name": "*", "update-types": ["version-update:semver-major", "version-update:semver-minor"]} in ( docker["ignore"]) + + +def test_a_disk_form_is_built_over_its_bases_published_image_with_the_published_tools_and_booted_before_its_push(): + """Its base's image, an agent image, and the disk tools, an infra image, are published first; its build is checked + and pushed as the infra images' are.""" + build = "\n".join(_runs("disk-build")) + + assert {"infra-publish", "agent-publish"} <= set(JOBS["disk-build"]["needs"]) + assert all(_finished(job) in _condition("disk-build") for job in ("infra-publish", "agent-publish")) + assert (build.index(f'{_SCRIPT} build "$ENTRY" "$ARCH"') < build.index("docker build -q -t agentenv-qemu-launcher ") + < build.index(f'{_SCRIPT} smoke "$NAME" "$BUILT" --launcher agentenv-qemu-launcher') + < build.index('docker push "$BUILT"')) diff --git a/tst/util/capabilities.py b/tst/util/capabilities.py index b75bff118..aca93d524 100644 --- a/tst/util/capabilities.py +++ b/tst/util/capabilities.py @@ -61,6 +61,10 @@ #: so collecting off a local sandbox needs GNU stat first on PATH. GNU_STAT = "gnu_stat" +#: Booting a computer's disk form on local Docker needs KVM: /dev/kvm on this host, which Docker passes to the launcher. +#: Docker Desktop on a Mac has none, nor has a host without nested virtualization. +KVM = "kvm" + def missing_capability_reason(capability: str) -> str: return f"{MISSING_CAPABILITY_PREFIX}{capability}" @@ -194,3 +198,15 @@ def skip_without_gnu_stat() -> pytest.MarkDecorator: not gnu_stat_is_available(), reason=missing_capability_reason(GNU_STAT), ) + + +def kvm_is_available() -> bool: + """Whether the local sandbox provider boots disks here: this host's /dev/kvm, readable and writable.""" + from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider + + return bool(LocalSandboxProvider.disk_image_platforms()) + + +def skip_without_kvm() -> pytest.MarkDecorator: + """Collection-time ``skipif`` for tests that boot a computer's disk form on local Docker.""" + return pytest.mark.skipif(not kvm_is_available(), reason=missing_capability_reason(KVM)) diff --git a/tst/util/oci_registry.py b/tst/util/oci_registry.py index 8dc684d26..cea05dec3 100644 --- a/tst/util/oci_registry.py +++ b/tst/util/oci_registry.py @@ -22,7 +22,7 @@ def __init__(self): self.manifests, self.blobs, self.tags, self.fetched = {}, {}, {}, [] def serve(self, monkeypatch) -> "Registry": - for name in ("pin_digest", "read_manifest", "read_blob"): + for name in ("pin_digest", "read_manifest", "read_blob", "read_blob_start"): monkeypatch.setattr(image_contents, name, getattr(self, name)) return self @@ -63,6 +63,10 @@ def read_blob(self, ref, digest, auth, path): self.fetched.append(digest) path.write_bytes(self.blobs[digest]) + def read_blob_start(self, ref, digest, auth, size): + self.fetched.append(digest) + return self.blobs[digest][:size] + def layer(files: dict[str, bytes | str]) -> bytes: """A tar.gz layer of ``files`` by path; a str is a link to that path."""