From 9bd59a298851d34e5fc93d2599a2547f756b6424 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Tue, 1 Sep 2026 17:23:19 +0800
Subject: [PATCH 01/21] Separate v19 EBS population from AWS orchestration

Keep root filesystem population credential-free so the release controller can own AWS calls while the converter handles only its attached block device. Load AWS SDK dependencies only on legacy API paths, support NVMe partition names, and clean nested mounts after failure.

Install the reviewed digest-bound tklamq 0.12.1 candidate before signed-archive hubclient 0.3.0, remove the candidate file, and retain exact source and binary provenance without persistent unsigned APT state. Verified with the focused EC2 v19 sandbox suite.
---
 bin/ec2/ebs_bundle.py   |  70 +++++++++++---------
 bin/ec2/ebs_populate.py |  23 +++++++
 bin/ec2/utils.py        |  11 +++-
 patches/ec2/conf        |  21 +++++-
 tests/ec2-v19           | 137 ++++++++++++++++++++++++++++++++++++++++
 5 files changed, 227 insertions(+), 35 deletions(-)
 create mode 100755 bin/ec2/ebs_populate.py
 create mode 100755 tests/ec2-v19

diff --git a/bin/ec2/ebs_bundle.py b/bin/ec2/ebs_bundle.py
index d38472c..bd8e1a6 100755
--- a/bin/ec2/ebs_bundle.py
+++ b/bin/ec2/ebs_bundle.py
@@ -30,8 +30,6 @@
 import sys
 import time
 
-from botocore.exceptions import ClientError
-
 import utils
 
 log = utils.get_logger("ebs-bundle")
@@ -102,6 +100,8 @@ def create(self, size, zone=None):
 
     def delete(self, max_attempts=10):
         if self.vol:
+            from botocore.exceptions import ClientError
+
             attempt = 0
             while True:
                 attempt += 1
@@ -153,8 +153,8 @@ def __del__(self):
 
 
 class Device:
-    def __init__(self):
-        self.real_path = self._get_freedevice()
+    def __init__(self, real_path=None):
+        self.real_path = real_path if real_path else self._get_freedevice()
         if not self.real_path:
             raise EbsBundleError("no free devices available...")
 
@@ -197,13 +197,46 @@ def mkpart(self):
         subprocess.run(["partprobe", self.real_path], check=True)
         time.sleep(5)
         self.root_path = self.real_path
-        self.real_path = self.real_path + "2"
+        separator = "p" if os.path.basename(self.real_path).startswith("nvme") else ""
+        self.real_path = self.real_path + separator + "2"
 
     def __del__(self):
         if self.is_mounted():
             self.umount()
 
 
+def populate(rootfs, device, filesystem="ext4"):
+    """Populate one pre-attached block device without making an AWS call."""
+    log.info("creating partitions")
+    device.mkpart()
+    device.mkfs(filesystem)
+    mount_path = rootfs + ".mount"
+    device.mount(mount_path)
+
+    submounts = []
+    try:
+        log.info("syncing rootfs to partition")
+        utils.rsync(rootfs, mount_path)
+
+        log.info("installing GRUB on volume")
+        for submount in ("/sys", "/proc", "/dev"):
+            subprocess.run(
+                ["mount", "--bind", "--make-rslave", submount, mount_path + submount],
+                check=True,
+            )
+            submounts.append(submount)
+        subprocess.run(["chroot", mount_path, "grub-install", device.root_path], check=True)
+        subprocess.run(["chroot", mount_path, "update-grub"], check=True)
+        subprocess.run(["chroot", mount_path, "update-initramfs", "-u"], check=True)
+    finally:
+        for submount in reversed(submounts):
+            subprocess.run(["umount", "-l", mount_path + submount], check=False)
+        if device.is_mounted():
+            device.umount()
+        if os.path.isdir(mount_path):
+            os.rmdir(mount_path)
+
+
 def bundle(rootfs, snapshot_name, size=10, filesystem="ext4"):
     log.info(f"target snapshot - {snapshot_name} ")
 
@@ -214,33 +247,8 @@ def bundle(rootfs, snapshot_name, size=10, filesystem="ext4"):
     device = Device()
     volume.attach(utils.get_instanceid(), device)
 
-    log.info("creating partitions")
-    device.mkpart()
-    device.mkfs(filesystem)
-    mount_path = rootfs + ".mount"
-    device.mount(mount_path)
-
-    log.info("syncing rootfs to partition")
-    utils.rsync(rootfs, mount_path)
-
-    log.info("installing GRUB on volume")
-    submounts = ["/sys", "/proc", "/dev"]
-    for s in submounts:
-        subprocess.run(["mount", "--bind", "--make-rslave", s, mount_path + s],
-                       check=True)
-    subprocess.run(["chroot", mount_path, "grub-install", device.root_path],
-                   check=True)
-    subprocess.run(["chroot", mount_path, "update-grub"], check=True)
-    subprocess.run(["chroot", mount_path, "update-initramfs", "-u"],
-                   check=True)
-
-    submounts.reverse()
-    for s in submounts:
-        subprocess.run(["umount", "-l", mount_path + s], check=True)
-
-    device.umount()
+    populate(rootfs, device, filesystem)
     volume.detach()
-    os.removedirs(mount_path)
 
     log.info("creating snapshot from volume")
     snapshot = Snapshot()
diff --git a/bin/ec2/ebs_populate.py b/bin/ec2/ebs_populate.py
new file mode 100755
index 0000000..891dd0c
--- /dev/null
+++ b/bin/ec2/ebs_populate.py
@@ -0,0 +1,23 @@
+#!/usr/bin/python3
+"""Populate a pre-attached EBS device from a prepared root filesystem."""
+
+import argparse
+import os
+
+from ebs_bundle import Device, populate
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("rootfs")
+    parser.add_argument("device")
+    args = parser.parse_args()
+    if not os.path.isdir(args.rootfs):
+        parser.error("rootfs path does not exist")
+    if not os.path.exists(args.device):
+        parser.error("device path does not exist")
+    populate(args.rootfs, Device(args.device))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bin/ec2/utils.py b/bin/ec2/utils.py
index 4c6b76c..6545e36 100644
--- a/bin/ec2/utils.py
+++ b/bin/ec2/utils.py
@@ -14,13 +14,12 @@
 import subprocess
 import sys
 
-import boto3
-import ec2metadata
-
 import conf
 
 
 def connect_boto3(region=None):
+    import boto3
+
     region = region if region else get_region()
     return boto3.client(
         "ec2",
@@ -35,14 +34,20 @@ def get_turnkey_version(rootfs):
 
 
 def get_instanceid():
+    import ec2metadata
+
     return ec2metadata.get("instance-id")
 
 
 def get_zone():
+    import ec2metadata
+
     return ec2metadata.get("availability-zone")
 
 
 def get_region():
+    import ec2metadata
+
     return ec2metadata.get("availability-zone")[0:-1]
 
 
diff --git a/patches/ec2/conf b/patches/ec2/conf
index 06d5c37..7f35aa7 100755
--- a/patches/ec2/conf
+++ b/patches/ec2/conf
@@ -8,7 +8,26 @@ install() {
         install $@
 }
 
-install hubclient sudo parted gdisk #ebsmount
+tklamq_candidate=/root/tklamq_0.12.1_all.deb
+tklamq_sha256=d972012e6aae81ada59b071ede494f46194db706a6805ce048215b510a168a2d
+echo "$tklamq_sha256  $tklamq_candidate" | sha256sum --check --strict
+install "$tklamq_candidate"
+rm -f "$tklamq_candidate"
+
+# hubclient remains an exact signed-archive input. The local tklamq version is
+# a normal package version so a later signed archive release can supersede it.
+install hubclient=0.3.0 sudo parted gdisk #ebsmount
+test "$(dpkg-query -W -f='${Version}' tklamq)" = 0.12.1
+
+/usr/bin/install -d -m 0755 /usr/share/doc/tklamq
+cat > /usr/share/doc/tklamq/turnkey-v19-build-input.json <<'EOF'
+{
+  "artifact": "tklamq_0.12.1_all.deb",
+  "sha256": "d972012e6aae81ada59b071ede494f46194db706a6805ce048215b510a168a2d",
+  "source_commit": "85e3d33c2d943e34db2c17e05403ea4a0f756784",
+  "source_pr": "https://github.com/turnkeylinux/tklamq/pull/5"
+}
+EOF
 
 # grub tweaks
 DEFAULT=/etc/default/grub
diff --git a/tests/ec2-v19 b/tests/ec2-v19
new file mode 100755
index 0000000..6c1bc1e
--- /dev/null
+++ b/tests/ec2-v19
@@ -0,0 +1,137 @@
+#!/usr/bin/python3
+"""Focused checks for the credential-free v19 EBS population path."""
+
+import importlib
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+
+EC2_ROOT = Path(__file__).resolve().parents[1] / "bin/ec2"
+sys.path.insert(0, str(EC2_ROOT))
+sys.modules.setdefault("ec2metadata", mock.Mock())
+ebs_bundle = importlib.import_module("ebs_bundle")
+ebs_populate = importlib.import_module("ebs_populate")
+
+
+class FakeDevice:
+    def __init__(self):
+        self.root_path = "/dev/test"
+        self.mounted = False
+        self.calls = []
+
+    def mkpart(self):
+        self.calls.append("mkpart")
+
+    def mkfs(self, filesystem):
+        self.calls.append(("mkfs", filesystem))
+
+    def mount(self, path):
+        Path(path).mkdir()
+        self.mounted = True
+        self.calls.append(("mount", path))
+
+    def is_mounted(self):
+        return self.mounted
+
+    def umount(self):
+        self.mounted = False
+        self.calls.append("umount")
+
+
+class PopulateTests(unittest.TestCase):
+    def test_local_command_imports_without_aws_site_packages(self):
+        completed = subprocess.run(
+            [sys.executable, "-S", str(EC2_ROOT / "ebs_populate.py"), "--help"],
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        self.assertEqual(completed.returncode, 0, completed.stderr)
+
+    def test_ec2_patch_installs_reviewed_tklamq_before_signed_hubclient(self):
+        conf = (EC2_ROOT.parents[1] / "patches/ec2/conf").read_text()
+        digest = "d972012e6aae81ada59b071ede494f46194db706a6805ce048215b510a168a2d"
+        source_commit = "85e3d33c2d943e34db2c17e05403ea4a0f756784"
+
+        self.assertIn(digest, conf)
+        self.assertIn(source_commit, conf)
+        self.assertIn("https://github.com/turnkeylinux/tklamq/pull/5", conf)
+        candidate_install = conf.index('install "$tklamq_candidate"')
+        hubclient_install = conf.index("install hubclient=0.3.0")
+        self.assertLess(candidate_install, hubclient_install)
+        self.assertIn('rm -f "$tklamq_candidate"', conf)
+        self.assertIn("dpkg-query -W", conf)
+        self.assertIn("/usr/bin/install -d -m 0755 /usr/share/doc/tklamq", conf)
+        self.assertNotIn("\ninstall -d", conf)
+        for forbidden in ("trusted=yes", "sources.list", "preferences.d", "GH_TOKEN", "AWS_"):
+            self.assertNotIn(forbidden, conf)
+
+    def test_nvme_partition_path_uses_p_separator(self):
+        device = ebs_bundle.Device("/dev/nvme1n1")
+        with mock.patch.object(ebs_bundle.subprocess, "run") as run, \
+                mock.patch.object(ebs_bundle.time, "sleep"):
+            device.mkpart()
+        self.assertEqual(device.root_path, "/dev/nvme1n1")
+        self.assertEqual(device.real_path, "/dev/nvme1n1p2")
+        self.assertEqual(run.call_count, 2)
+
+    def test_populates_and_unmounts_without_aws(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            rootfs = str(Path(temporary) / "rootfs")
+            Path(rootfs).mkdir()
+            device = FakeDevice()
+            commands = []
+
+            def run(command, **kwargs):
+                commands.append((command, kwargs))
+                return mock.Mock(returncode=0)
+
+            with mock.patch.object(ebs_bundle.subprocess, "run", side_effect=run), \
+                    mock.patch.object(ebs_bundle.utils, "rsync") as rsync:
+                ebs_bundle.populate(rootfs, device)
+
+            self.assertEqual(device.calls[:2], ["mkpart", ("mkfs", "ext4")])
+            self.assertEqual(device.calls[-1], "umount")
+            rsync.assert_called_once_with(rootfs, rootfs + ".mount")
+            self.assertIn((["chroot", rootfs + ".mount", "grub-install", "/dev/test"], {"check": True}), commands)
+            self.assertFalse(Path(rootfs + ".mount").exists())
+
+    def test_unmounts_after_chroot_failure(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            rootfs = str(Path(temporary) / "rootfs")
+            Path(rootfs).mkdir()
+            device = FakeDevice()
+
+            def run(command, **kwargs):
+                if command[:3] == ["chroot", rootfs + ".mount", "grub-install"]:
+                    raise RuntimeError("reproduced grub failure")
+                return mock.Mock(returncode=0)
+
+            with mock.patch.object(ebs_bundle.subprocess, "run", side_effect=run), \
+                    mock.patch.object(ebs_bundle.utils, "rsync"):
+                with self.assertRaisesRegex(RuntimeError, "reproduced grub failure"):
+                    ebs_bundle.populate(rootfs, device)
+
+            self.assertFalse(device.mounted)
+            self.assertFalse(Path(rootfs + ".mount").exists())
+
+    def test_command_uses_explicit_local_device(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            rootfs = Path(temporary) / "rootfs"
+            device = Path(temporary) / "device"
+            rootfs.mkdir()
+            device.touch()
+            with mock.patch.object(sys, "argv", ["ebs_populate.py", str(rootfs), str(device)]), \
+                    mock.patch.object(ebs_populate, "Device") as device_class, \
+                    mock.patch.object(ebs_populate, "populate") as populate:
+                ebs_populate.main()
+            device_class.assert_called_once_with(str(device))
+            populate.assert_called_once_with(str(rootfs), device_class.return_value)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

From d179aca22ffd92640148b8061ebb771fce54f406 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Tue, 1 Sep 2026 20:16:43 +0800
Subject: [PATCH 02/21] Use the built root for TKLBAM profiles

Keep the existing host interpreter path when it is available. When an older build host lacks that runtime, execute the completed v19 root filesystem's relocatable PyPy against the root and load its TKLBAM libraries, avoiding incompatible host package installation.

Add a focused LibreSBX fixture covering host selection, root-runtime output, filtered profile contents, absence of staging residue, and nonzero failure propagation.
---
 bin/iso-release                          |  10 +-
 tests/test-iso-release-tklbam-runtime.sh | 201 +++++++++++++++++++++++
 2 files changed, 210 insertions(+), 1 deletion(-)
 create mode 100755 tests/test-iso-release-tklbam-runtime.sh

diff --git a/bin/iso-release b/bin/iso-release
index b8ac922..05e49a4 100755
--- a/bin/iso-release
+++ b/bin/iso-release
@@ -90,7 +90,15 @@ cp build/product.iso "$O/$name.iso"
 if [[ -e $BT_PROFILES/$appname ]]; then
     mkdir -p "$O/$name.tklbam"
     export PROFILES_CONF=$BT_PROFILES
-    "$BT/bin/generate-tklbam-profile" "$O/$name.iso" "$O/$name.tklbam"
+    if [[ -x /usr/lib/tklbam-pypy2/bin/pypy ]]; then
+        "$BT/bin/generate-tklbam-profile" "$O/$name.iso" "$O/$name.tklbam"
+    else
+        rootfs=$(readlink -f "$rootfs")
+        PROFILES_CONF="$BT_PROFILES" \
+        TKLBAM_LIB_PATH="$rootfs/usr/lib/tklbam" \
+            "$rootfs/usr/lib/tklbam-pypy2/bin/pypy" \
+            "$BT/bin/generate-tklbam-profile" "$rootfs" "$O/$name.tklbam"
+    fi
 fi
 if [[ -z "$no_screens" ]]; then
     mkdir -p "$O/$name.screens"
diff --git a/tests/test-iso-release-tklbam-runtime.sh b/tests/test-iso-release-tklbam-runtime.sh
new file mode 100755
index 0000000..922c51e
--- /dev/null
+++ b/tests/test-iso-release-tklbam-runtime.sh
@@ -0,0 +1,201 @@
+#!/bin/bash
+set -Eeuo pipefail
+
+[[ $EUID -eq 0 ]] || {
+    echo "this fixture must run as root inside LibreSBX" >&2
+    exit 1
+}
+
+repo=$(cd "$(dirname "$0")/.." && pwd)
+fixture=$(mktemp -d)
+host_pypy=/usr/lib/tklbam-pypy2/bin/pypy
+host_parent_created=
+host_pypy_created=
+
+cleanup() {
+    if [[ -n "$host_pypy_created" ]]; then
+        rm -f -- "$host_pypy"
+    fi
+    if [[ -n "$host_parent_created" ]]; then
+        rmdir /usr/lib/tklbam-pypy2/bin /usr/lib/tklbam-pypy2 2>/dev/null || true
+    fi
+    rm -rf -- "$fixture"
+}
+trap cleanup EXIT
+
+[[ ! -e "$host_pypy" ]] || {
+    echo "$host_pypy already exists in the test sandbox" >&2
+    exit 1
+}
+
+bt=$fixture/buildtasks
+profiles=$fixture/profiles
+commands=$fixture/bin
+log=$fixture/calls
+name=turnkey-tkldev-19.0-trixie-amd64
+install -d "$bt/bin" "$bt/config" "$profiles" "$commands"
+cp "$repo/bin/iso-release" "$repo/bin/generate-tklbam-profile" "$bt/bin/"
+printf 'core profile\n' > "$profiles/core"
+printf 'tkldev profile\n' > "$profiles/tkldev"
+
+cat > "$bt/config/common.cfg" <<'EOF'
+export BT_PROFILES=${TEST_BT_PROFILES:?}
+EOF
+
+for helper in generate-signature generate-manifest generate-buildenv; do
+    cat > "$bt/bin/$helper" <<'EOF'
+#!/bin/bash
+case "$(basename "$0")" in
+    generate-manifest) echo manifest ;;
+    generate-buildenv) echo buildenv ;;
+esac
+EOF
+    chmod 0755 "$bt/bin/$helper"
+done
+
+cat > "$commands/turnkey-version" <<EOF
+#!/bin/bash
+if [[ " \$* " == *" --name "* ]]; then
+    echo tkldev
+else
+    echo $name
+fi
+EOF
+chmod 0755 "$commands/turnkey-version"
+
+make_product() {
+    local case_name=$1
+    local product=$fixture/$case_name/product
+    local rootfs=$product/build/root.sandbox
+    install -d "$rootfs/usr/lib/tklbam-pypy2/bin" \
+        "$rootfs/usr/lib/tklbam" "$rootfs/var/lib/dpkg" \
+        "$fixture/$case_name/output"
+    printf 'changelog\n' > "$product/changelog"
+    printf 'iso\n' > "$product/build/product.iso"
+    cat > "$rootfs/var/lib/dpkg/status" <<'EOF'
+Package: bash
+Status: install ok installed
+
+Package: live-tools
+Status: install ok installed
+
+Package: tkl-installer
+Status: install ok installed
+
+Package: removed-package
+Status: deinstall ok config-files
+EOF
+    cat > "$rootfs/usr/lib/tklbam-pypy2/bin/pypy" <<'EOF'
+#!/bin/bash
+set -eu
+
+generator=$1
+rootfs=$2
+output=$3
+[[ $generator == "$TEST_GENERATOR" ]]
+[[ $PROFILES_CONF == "$TEST_PROFILES" ]]
+[[ $TKLBAM_LIB_PATH == "$rootfs/usr/lib/tklbam" ]]
+printf 'root:%s:%s\n' "$rootfs" "$output" >> "$TEST_LOG"
+if [[ -n ${TEST_ROOT_RUNTIME_FAIL:-} ]]; then
+    exit "$TEST_ROOT_RUNTIME_FAIL"
+fi
+
+archive_root=$(mktemp -d)
+cleanup_archive() {
+    rm -rf -- "$archive_root"
+}
+trap cleanup_archive EXIT
+printf 'dirindex\n' > "$archive_root/dirindex"
+printf 'dirindex conf\n' > "$archive_root/dirindex.conf"
+awk '
+    /^Package: / { package = $2 }
+    /^Status: install ok installed$/ &&
+        package !~ /^(di-live|tkl-installer|live-boot|live-boot-initramfs-tools|live-tools)$/ {
+            print package
+        }
+' "$rootfs/var/lib/dpkg/status" | sort > "$archive_root/packages"
+tar -C "$archive_root" -zcf "$output/$TEST_NAME.tar.gz" .
+EOF
+    chmod 0755 "$rootfs/usr/lib/tklbam-pypy2/bin/pypy"
+    printf '%s\n' "$product"
+}
+
+run_release() {
+    local product=$1
+    local output=$2
+    local runtime_fail=${3:-}
+    (
+        cd "$product"
+        export PATH="$commands:$PATH"
+        export TEST_BT_PROFILES=$profiles
+        export TEST_GENERATOR=$bt/bin/generate-tklbam-profile
+        export TEST_LOG=$log
+        export TEST_NAME=$name
+        export TEST_PROFILES=$profiles
+        export TEST_ROOT_RUNTIME_FAIL=$runtime_fail
+        "$bt/bin/iso-release" --no-screens "$output"
+    )
+}
+
+install_host_pypy() {
+    if [[ ! -d /usr/lib/tklbam-pypy2 ]]; then
+        host_parent_created=yes
+    fi
+    install -d /usr/lib/tklbam-pypy2/bin
+    cat > "$host_pypy" <<'EOF'
+#!/bin/bash
+set -eu
+printf 'host:%s\n' "$*" >> "$TEST_LOG"
+[[ $1 == "$TEST_GENERATOR" ]]
+archive_root=$(mktemp -d)
+trap 'rm -rf -- "$archive_root"' EXIT
+printf 'host runtime\n' > "$archive_root/runtime"
+tar -C "$archive_root" -zcf "$3/$TEST_NAME.tar.gz" .
+EOF
+    chmod 0755 "$host_pypy"
+    host_pypy_created=yes
+}
+
+remove_host_pypy() {
+    rm -f -- "$host_pypy"
+    host_pypy_created=
+    if [[ -n "$host_parent_created" ]]; then
+        rmdir /usr/lib/tklbam-pypy2/bin /usr/lib/tklbam-pypy2
+        host_parent_created=
+    fi
+}
+
+: > "$log"
+install_host_pypy
+host_product=$(make_product host)
+run_release "$host_product" "$fixture/host/output"
+grep -q '^host:' "$log"
+! grep -q '^root:' "$log"
+[[ -f "$fixture/host/output/$name.tklbam/$name.tar.gz" ]]
+remove_host_pypy
+
+: > "$log"
+root_product=$(make_product root)
+run_release "$root_product" "$fixture/root/output"
+grep -q '^root:' "$log"
+archive=$fixture/root/output/$name.tklbam/$name.tar.gz
+[[ -f "$archive" ]]
+tar -tzf "$archive" | grep -qx './dirindex'
+tar -tzf "$archive" | grep -qx './dirindex.conf'
+tar -tzf "$archive" | grep -qx './packages'
+tar -xOzf "$archive" ./packages | grep -qx bash
+! tar -xOzf "$archive" ./packages | grep -Eq '^(live-tools|tkl-installer)$'
+! find "$root_product/build/root.sandbox" -name 'buildtasks-tklbam-profile.*' \
+    -print -quit | grep -q .
+
+: > "$log"
+failure_product=$(make_product failure)
+set +e
+run_release "$failure_product" "$fixture/failure/output" 37
+status=$?
+set -e
+[[ $status -eq 37 ]]
+grep -q '^root:' "$log"
+[[ ! -e "$fixture/failure/output/$name.tklbam/$name.tar.gz" ]]
+! find "$failure_product/build/root.sandbox" -name 'buildtasks-tklbam-profile.*' \
+    -print -quit | grep -q .

From e71bbdda6d9753b1758348450ee63881cc5aed4a Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Tue, 1 Sep 2026 20:41:27 +0800
Subject: [PATCH 03/21] Run TKLBAM profiles inside the built root

Frozen TKLDev v18 workers cannot load the Trixie PyPy runtime directly because it requires a newer glibc. Keep the existing host-runtime path, but run the fallback through chroot so the generator uses the libraries from the completed v19 root.

Stage only the generator and selected profile inputs inside a unique root-local directory, return the generated archive, and clean the stage after successful generation, generator failure, or output-copy failure. The focused runtime test covers selection, arguments, filtering, cleanup, and exact failure propagation.
---
 bin/iso-release                          |  39 ++++++-
 tests/test-iso-release-tklbam-runtime.sh | 123 +++++++++++++++--------
 2 files changed, 117 insertions(+), 45 deletions(-)

diff --git a/bin/iso-release b/bin/iso-release
index 05e49a4..06b97c6 100755
--- a/bin/iso-release
+++ b/bin/iso-release
@@ -93,11 +93,40 @@ if [[ -e $BT_PROFILES/$appname ]]; then
     if [[ -x /usr/lib/tklbam-pypy2/bin/pypy ]]; then
         "$BT/bin/generate-tklbam-profile" "$O/$name.iso" "$O/$name.tklbam"
     else
-        rootfs=$(readlink -f "$rootfs")
-        PROFILES_CONF="$BT_PROFILES" \
-        TKLBAM_LIB_PATH="$rootfs/usr/lib/tklbam" \
-            "$rootfs/usr/lib/tklbam-pypy2/bin/pypy" \
-            "$BT/bin/generate-tklbam-profile" "$rootfs" "$O/$name.tklbam"
+        (
+            rootfs=$(readlink -f "$rootfs")
+            [[ -x "$rootfs/usr/lib/tklbam-pypy2/bin/pypy" ]] || \
+                fatal "TKLBAM runtime not found in $rootfs"
+            staging=$(mktemp -d "$rootfs/var/tmp/buildtasks-tklbam-profile.XXXXXX")
+            guest_staging=${staging#"$rootfs"}
+            cleanup() {
+                status=$?
+                trap - EXIT
+                rm -rf -- "$staging"
+                exit "$status"
+            }
+            trap cleanup EXIT
+
+            install -d "$staging/profiles" "$staging/output"
+            install -m 0755 "$BT/bin/generate-tklbam-profile" \
+                "$staging/generate-tklbam-profile"
+            cp "$BT_PROFILES/core" "$staging/profiles/core"
+            if [[ "$appname" != "core" ]]; then
+                cp "$BT_PROFILES/$appname" "$staging/profiles/$appname"
+            fi
+            if [[ -d "$BT_PROFILES/hooks/$appname" ]]; then
+                install -d "$staging/profiles/hooks"
+                cp -a "$BT_PROFILES/hooks/$appname" "$staging/profiles/hooks/"
+            fi
+
+            chroot "$rootfs" /usr/bin/env \
+                PROFILES_CONF="$guest_staging/profiles" \
+                TKLBAM_LIB_PATH=/usr/lib/tklbam \
+                /usr/lib/tklbam-pypy2/bin/pypy \
+                "$guest_staging/generate-tklbam-profile" / \
+                "$guest_staging/output"
+            cp -a "$staging/output/." "$O/$name.tklbam/"
+        )
     fi
 fi
 if [[ -z "$no_screens" ]]; then
diff --git a/tests/test-iso-release-tklbam-runtime.sh b/tests/test-iso-release-tklbam-runtime.sh
index 922c51e..ddab569 100755
--- a/tests/test-iso-release-tklbam-runtime.sh
+++ b/tests/test-iso-release-tklbam-runtime.sh
@@ -33,10 +33,11 @@ profiles=$fixture/profiles
 commands=$fixture/bin
 log=$fixture/calls
 name=turnkey-tkldev-19.0-trixie-amd64
-install -d "$bt/bin" "$bt/config" "$profiles" "$commands"
+install -d "$bt/bin" "$bt/config" "$profiles/hooks/tkldev" "$commands"
 cp "$repo/bin/iso-release" "$repo/bin/generate-tklbam-profile" "$bt/bin/"
 printf 'core profile\n' > "$profiles/core"
 printf 'tkldev profile\n' > "$profiles/tkldev"
+printf 'profile hook\n' > "$profiles/hooks/tkldev/profile-hook"
 
 cat > "$bt/config/common.cfg" <<'EOF'
 export BT_PROFILES=${TEST_BT_PROFILES:?}
@@ -63,13 +64,71 @@ fi
 EOF
 chmod 0755 "$commands/turnkey-version"
 
+cat > "$commands/chroot" <<'EOF'
+#!/bin/bash
+set -eu
+
+rootfs=$1
+shift
+[[ $1 == /usr/bin/env ]]
+shift
+profiles_arg=$1
+shift
+[[ $profiles_arg == PROFILES_CONF=* ]]
+guest_profiles=${profiles_arg#PROFILES_CONF=}
+[[ $1 == TKLBAM_LIB_PATH=/usr/lib/tklbam ]]
+shift
+[[ $1 == /usr/lib/tklbam-pypy2/bin/pypy ]]
+shift
+guest_generator=$1
+root_arg=$2
+guest_output=$3
+[[ $root_arg == / ]]
+[[ -x $rootfs/usr/lib/tklbam-pypy2/bin/pypy ]]
+
+cmp -s "$TEST_GENERATOR" "$rootfs$guest_generator"
+cmp -s "$TEST_PROFILES/core" "$rootfs$guest_profiles/core"
+cmp -s "$TEST_PROFILES/tkldev" "$rootfs$guest_profiles/tkldev"
+cmp -s "$TEST_PROFILES/hooks/tkldev/profile-hook" \
+    "$rootfs$guest_profiles/hooks/tkldev/profile-hook"
+printf 'root:%s:%s\n' "$guest_generator" "$guest_output" >> "$TEST_LOG"
+if [[ -n ${TEST_CHROOT_FAIL:-} ]]; then
+    exit "$TEST_CHROOT_FAIL"
+fi
+
+archive_root=$(mktemp -d)
+trap 'rm -rf -- "$archive_root"' EXIT
+printf 'dirindex\n' > "$archive_root/dirindex"
+printf 'dirindex conf\n' > "$archive_root/dirindex.conf"
+awk '
+    /^Package: / { package = $2 }
+    /^Status: install ok installed$/ &&
+        package !~ /^(di-live|tkl-installer|live-boot|live-boot-initramfs-tools|live-tools)$/ {
+            print package
+        }
+' "$rootfs/var/lib/dpkg/status" | sort > "$archive_root/packages"
+tar -C "$archive_root" -zcf "$rootfs$guest_output/$TEST_NAME.tar.gz" .
+EOF
+chmod 0755 "$commands/chroot"
+
+cat > "$commands/cp" <<'EOF'
+#!/bin/bash
+set -eu
+if [[ -n ${TEST_COPY_FAIL:-} && " $* " == *'/buildtasks-tklbam-profile.'*'/output/. '* ]]; then
+    exit "$TEST_COPY_FAIL"
+fi
+exec /bin/cp "$@"
+EOF
+chmod 0755 "$commands/cp"
+
 make_product() {
     local case_name=$1
     local product=$fixture/$case_name/product
     local rootfs=$product/build/root.sandbox
     install -d "$rootfs/usr/lib/tklbam-pypy2/bin" \
-        "$rootfs/usr/lib/tklbam" "$rootfs/var/lib/dpkg" \
+        "$rootfs/usr/lib/tklbam" "$rootfs/var/lib/dpkg" "$rootfs/var/tmp" \
         "$fixture/$case_name/output"
+    install -m 0755 /bin/true "$rootfs/usr/lib/tklbam-pypy2/bin/pypy"
     printf 'changelog\n' > "$product/changelog"
     printf 'iso\n' > "$product/build/product.iso"
     cat > "$rootfs/var/lib/dpkg/status" <<'EOF'
@@ -85,54 +144,24 @@ Status: install ok installed
 Package: removed-package
 Status: deinstall ok config-files
 EOF
-    cat > "$rootfs/usr/lib/tklbam-pypy2/bin/pypy" <<'EOF'
-#!/bin/bash
-set -eu
-
-generator=$1
-rootfs=$2
-output=$3
-[[ $generator == "$TEST_GENERATOR" ]]
-[[ $PROFILES_CONF == "$TEST_PROFILES" ]]
-[[ $TKLBAM_LIB_PATH == "$rootfs/usr/lib/tklbam" ]]
-printf 'root:%s:%s\n' "$rootfs" "$output" >> "$TEST_LOG"
-if [[ -n ${TEST_ROOT_RUNTIME_FAIL:-} ]]; then
-    exit "$TEST_ROOT_RUNTIME_FAIL"
-fi
-
-archive_root=$(mktemp -d)
-cleanup_archive() {
-    rm -rf -- "$archive_root"
-}
-trap cleanup_archive EXIT
-printf 'dirindex\n' > "$archive_root/dirindex"
-printf 'dirindex conf\n' > "$archive_root/dirindex.conf"
-awk '
-    /^Package: / { package = $2 }
-    /^Status: install ok installed$/ &&
-        package !~ /^(di-live|tkl-installer|live-boot|live-boot-initramfs-tools|live-tools)$/ {
-            print package
-        }
-' "$rootfs/var/lib/dpkg/status" | sort > "$archive_root/packages"
-tar -C "$archive_root" -zcf "$output/$TEST_NAME.tar.gz" .
-EOF
-    chmod 0755 "$rootfs/usr/lib/tklbam-pypy2/bin/pypy"
     printf '%s\n' "$product"
 }
 
 run_release() {
     local product=$1
     local output=$2
-    local runtime_fail=${3:-}
+    local chroot_fail=${3:-}
+    local copy_fail=${4:-}
     (
         cd "$product"
         export PATH="$commands:$PATH"
         export TEST_BT_PROFILES=$profiles
+        export TEST_CHROOT_FAIL=$chroot_fail
+        export TEST_COPY_FAIL=$copy_fail
         export TEST_GENERATOR=$bt/bin/generate-tklbam-profile
         export TEST_LOG=$log
         export TEST_NAME=$name
         export TEST_PROFILES=$profiles
-        export TEST_ROOT_RUNTIME_FAIL=$runtime_fail
         "$bt/bin/iso-release" --no-screens "$output"
     )
 }
@@ -165,6 +194,11 @@ remove_host_pypy() {
     fi
 }
 
+assert_no_stage() {
+    ! find "$1/build/root.sandbox/var/tmp" -mindepth 1 -maxdepth 1 \
+        -name 'buildtasks-tklbam-profile.*' -print -quit | grep -q .
+}
+
 : > "$log"
 install_host_pypy
 host_product=$(make_product host)
@@ -185,8 +219,7 @@ tar -tzf "$archive" | grep -qx './dirindex.conf'
 tar -tzf "$archive" | grep -qx './packages'
 tar -xOzf "$archive" ./packages | grep -qx bash
 ! tar -xOzf "$archive" ./packages | grep -Eq '^(live-tools|tkl-installer)$'
-! find "$root_product/build/root.sandbox" -name 'buildtasks-tklbam-profile.*' \
-    -print -quit | grep -q .
+assert_no_stage "$root_product"
 
 : > "$log"
 failure_product=$(make_product failure)
@@ -197,5 +230,15 @@ set -e
 [[ $status -eq 37 ]]
 grep -q '^root:' "$log"
 [[ ! -e "$fixture/failure/output/$name.tklbam/$name.tar.gz" ]]
-! find "$failure_product/build/root.sandbox" -name 'buildtasks-tklbam-profile.*' \
-    -print -quit | grep -q .
+assert_no_stage "$failure_product"
+
+: > "$log"
+copy_product=$(make_product copy-failure)
+set +e
+run_release "$copy_product" "$fixture/copy-failure/output" '' 41
+status=$?
+set -e
+[[ $status -eq 41 ]]
+grep -q '^root:' "$log"
+[[ ! -e "$fixture/copy-failure/output/$name.tklbam/$name.tar.gz" ]]
+assert_no_stage "$copy_product"

From 0c04fc26449d055067857e85a1dda93230292dbc Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Tue, 1 Sep 2026 21:02:34 +0800
Subject: [PATCH 04/21] Expose the packaged PyPy library in chroot

The Trixie turnkey-pypy2 package installs libpypy-c.so beside its interpreter, outside the default loader search path. The v19-root fallback reached the chroot correctly but failed before running the profile generator because that library was not found.

Add only the exact packaged directory to the fallback chroot environment. The host-runtime branch is unchanged, and the focused test now requires the loader path while retaining generation and cleanup failure coverage.
---
 bin/iso-release                          | 1 +
 tests/test-iso-release-tklbam-runtime.sh | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/bin/iso-release b/bin/iso-release
index 06b97c6..efd7af5 100755
--- a/bin/iso-release
+++ b/bin/iso-release
@@ -122,6 +122,7 @@ if [[ -e $BT_PROFILES/$appname ]]; then
             chroot "$rootfs" /usr/bin/env \
                 PROFILES_CONF="$guest_staging/profiles" \
                 TKLBAM_LIB_PATH=/usr/lib/tklbam \
+                LD_LIBRARY_PATH=/usr/lib/tklbam-pypy2/bin \
                 /usr/lib/tklbam-pypy2/bin/pypy \
                 "$guest_staging/generate-tklbam-profile" / \
                 "$guest_staging/output"
diff --git a/tests/test-iso-release-tklbam-runtime.sh b/tests/test-iso-release-tklbam-runtime.sh
index ddab569..3e9d6ca 100755
--- a/tests/test-iso-release-tklbam-runtime.sh
+++ b/tests/test-iso-release-tklbam-runtime.sh
@@ -78,6 +78,8 @@ shift
 guest_profiles=${profiles_arg#PROFILES_CONF=}
 [[ $1 == TKLBAM_LIB_PATH=/usr/lib/tklbam ]]
 shift
+[[ $1 == LD_LIBRARY_PATH=/usr/lib/tklbam-pypy2/bin ]]
+shift
 [[ $1 == /usr/lib/tklbam-pypy2/bin/pypy ]]
 shift
 guest_generator=$1

From 721f55cd7e143d2873c5cdd3721999bd8e3a1b38 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Tue, 1 Sep 2026 21:21:39 +0800
Subject: [PATCH 05/21] Run profile generation through fab-chroot

The raw chroot reached the packaged v19 PyPy but aborted without the normal build runtime mounts. TKLDev already provides fab-chroot for commands that need the completed root environment.

Use its status-preserving no-argument script path with a root-local wrapper that only execs the staged generator. Pass the exact three required environment values, retain scoped cleanup, and cover the installed one-command boundary plus generator and output-copy failures in the focused test.
---
 bin/iso-release                          | 18 +++---
 tests/test-iso-release-tklbam-runtime.sh | 75 ++++++++++++++++--------
 2 files changed, 60 insertions(+), 33 deletions(-)

diff --git a/bin/iso-release b/bin/iso-release
index efd7af5..7d45e95 100755
--- a/bin/iso-release
+++ b/bin/iso-release
@@ -110,6 +110,11 @@ if [[ -e $BT_PROFILES/$appname ]]; then
             install -d "$staging/profiles" "$staging/output"
             install -m 0755 "$BT/bin/generate-tklbam-profile" \
                 "$staging/generate-tklbam-profile"
+            cat > "$staging/run-tklbam-profile" <<EOF
+#!/bin/bash
+exec /usr/lib/tklbam-pypy2/bin/pypy "$guest_staging/generate-tklbam-profile" / "$guest_staging/output"
+EOF
+            chmod 0755 "$staging/run-tklbam-profile"
             cp "$BT_PROFILES/core" "$staging/profiles/core"
             if [[ "$appname" != "core" ]]; then
                 cp "$BT_PROFILES/$appname" "$staging/profiles/$appname"
@@ -119,13 +124,12 @@ if [[ -e $BT_PROFILES/$appname ]]; then
                 cp -a "$BT_PROFILES/hooks/$appname" "$staging/profiles/hooks/"
             fi
 
-            chroot "$rootfs" /usr/bin/env \
-                PROFILES_CONF="$guest_staging/profiles" \
-                TKLBAM_LIB_PATH=/usr/lib/tklbam \
-                LD_LIBRARY_PATH=/usr/lib/tklbam-pypy2/bin \
-                /usr/lib/tklbam-pypy2/bin/pypy \
-                "$guest_staging/generate-tklbam-profile" / \
-                "$guest_staging/output"
+            PROFILES_CONF="$guest_staging/profiles" \
+            TKLBAM_LIB_PATH=/usr/lib/tklbam \
+            LD_LIBRARY_PATH=/usr/lib/tklbam-pypy2/bin \
+                fab-chroot \
+                -e PROFILES_CONF:TKLBAM_LIB_PATH:LD_LIBRARY_PATH \
+                "$rootfs" --script "$staging/run-tklbam-profile"
             cp -a "$staging/output/." "$O/$name.tklbam/"
         )
     fi
diff --git a/tests/test-iso-release-tklbam-runtime.sh b/tests/test-iso-release-tklbam-runtime.sh
index 3e9d6ca..7e6b903 100755
--- a/tests/test-iso-release-tklbam-runtime.sh
+++ b/tests/test-iso-release-tklbam-runtime.sh
@@ -64,42 +64,49 @@ fi
 EOF
 chmod 0755 "$commands/turnkey-version"
 
-cat > "$commands/chroot" <<'EOF'
+cat > "$commands/fab-chroot" <<'EOF'
 #!/bin/bash
 set -eu
 
-rootfs=$1
-shift
-[[ $1 == /usr/bin/env ]]
-shift
-profiles_arg=$1
-shift
-[[ $profiles_arg == PROFILES_CONF=* ]]
-guest_profiles=${profiles_arg#PROFILES_CONF=}
-[[ $1 == TKLBAM_LIB_PATH=/usr/lib/tklbam ]]
-shift
-[[ $1 == LD_LIBRARY_PATH=/usr/lib/tklbam-pypy2/bin ]]
-shift
-[[ $1 == /usr/lib/tklbam-pypy2/bin/pypy ]]
-shift
-guest_generator=$1
-root_arg=$2
-guest_output=$3
-[[ $root_arg == / ]]
+[[ $# -eq 5 ]]
+[[ $1 == -e ]]
+[[ $2 == PROFILES_CONF:TKLBAM_LIB_PATH:LD_LIBRARY_PATH ]]
+rootfs=$3
+[[ $4 == --script ]]
+wrapper=$5
+[[ $PROFILES_CONF == /var/tmp/buildtasks-tklbam-profile.*/profiles ]]
+[[ $TKLBAM_LIB_PATH == /usr/lib/tklbam ]]
+[[ $LD_LIBRARY_PATH == /usr/lib/tklbam-pypy2/bin ]]
+stage=${PROFILES_CONF%/profiles}
+guest_profiles=$PROFILES_CONF
+guest_generator=$stage/generate-tklbam-profile
+guest_output=$stage/output
+[[ $wrapper == "$rootfs$stage/run-tklbam-profile" ]]
 [[ -x $rootfs/usr/lib/tklbam-pypy2/bin/pypy ]]
 
-cmp -s "$TEST_GENERATOR" "$rootfs$guest_generator"
 cmp -s "$TEST_PROFILES/core" "$rootfs$guest_profiles/core"
 cmp -s "$TEST_PROFILES/tkldev" "$rootfs$guest_profiles/tkldev"
 cmp -s "$TEST_PROFILES/hooks/tkldev/profile-hook" \
     "$rootfs$guest_profiles/hooks/tkldev/profile-hook"
-printf 'root:%s:%s\n' "$guest_generator" "$guest_output" >> "$TEST_LOG"
-if [[ -n ${TEST_CHROOT_FAIL:-} ]]; then
-    exit "$TEST_CHROOT_FAIL"
-fi
+
+expected=$(mktemp)
+mapped=$(mktemp)
+cleanup_fab() {
+    rm -f -- "$expected" "$mapped"
+}
+trap cleanup_fab EXIT
+cat > "$expected" <<EXPECTED
+#!/bin/bash
+exec /usr/lib/tklbam-pypy2/bin/pypy "$guest_generator" / "$guest_output"
+EXPECTED
+cmp -s "$expected" "$wrapper"
+sed "s|^exec /usr/lib/tklbam-pypy2/bin/pypy |exec $TEST_PYPY |" \
+    "$wrapper" > "$mapped"
+chmod 0755 "$mapped"
+TEST_ROOTFS=$rootfs "$mapped"
 
 archive_root=$(mktemp -d)
-trap 'rm -rf -- "$archive_root"' EXIT
+trap 'rm -rf -- "$archive_root"; cleanup_fab' EXIT
 printf 'dirindex\n' > "$archive_root/dirindex"
 printf 'dirindex conf\n' > "$archive_root/dirindex.conf"
 awk '
@@ -111,7 +118,22 @@ awk '
 ' "$rootfs/var/lib/dpkg/status" | sort > "$archive_root/packages"
 tar -C "$archive_root" -zcf "$rootfs$guest_output/$TEST_NAME.tar.gz" .
 EOF
-chmod 0755 "$commands/chroot"
+chmod 0755 "$commands/fab-chroot"
+
+cat > "$commands/root-pypy" <<'EOF'
+#!/bin/bash
+set -eu
+
+guest_generator=$1
+[[ $2 == / ]]
+guest_output=$3
+cmp -s "$TEST_GENERATOR" "$TEST_ROOTFS$guest_generator"
+printf 'root:%s:%s\n' "$guest_generator" "$guest_output" >> "$TEST_LOG"
+if [[ -n ${TEST_CHROOT_FAIL:-} ]]; then
+    exit "$TEST_CHROOT_FAIL"
+fi
+EOF
+chmod 0755 "$commands/root-pypy"
 
 cat > "$commands/cp" <<'EOF'
 #!/bin/bash
@@ -163,6 +185,7 @@ run_release() {
         export TEST_GENERATOR=$bt/bin/generate-tklbam-profile
         export TEST_LOG=$log
         export TEST_NAME=$name
+        export TEST_PYPY=$commands/root-pypy
         export TEST_PROFILES=$profiles
         "$bt/bin/iso-release" --no-screens "$output"
     )

From ee4406a3085b57340f52e8e340eab7330a9ec3f5 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 01:51:01 +0800
Subject: [PATCH 06/21] Authorize EC2 keys for the active login user

EC2 images enable sudoadmin for ordinary non-Hub launches, but the firstboot key hook still wrote the instance key to root. Neither root nor admin could authenticate to a fresh canonical TKLDev v19 builder even though sshd was healthy.

Select admin when the established SUDOADMIN setting is true and preserve root for Hub-style launches where sudoadmin is disabled. Focused EC2 tests cover true, case-insensitive true, false, and missing configuration values.
---
 .../lib/inithooks/firstboot.d/40ec2-sshkeys   | 14 +++++++++--
 tests/ec2-v19                                 | 23 +++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys b/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys
index c1ea71b..6fc20fd 100755
--- a/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys
+++ b/patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys
@@ -10,7 +10,17 @@ if '_TURNKEY_INIT' in os.environ:
 import pwd
 import ec2metadata
 
-USERNAME = 'root'
+
+def login_username(config='/etc/default/inithooks'):
+    username = 'root'
+    with open(config) as fob:
+        for line in fob:
+            key, separator, value = line.partition('=')
+            if separator and key.strip() == 'SUDOADMIN':
+                if value.strip().strip("'\"").lower() == 'true':
+                    username = 'admin'
+                break
+    return username
 
 
 def authorize_sshkeys(keys, username):
@@ -31,7 +41,7 @@ def authorize_sshkeys(keys, username):
 def main():
     keys = ec2metadata.get('public-keys')
     if keys:
-        authorize_sshkeys(keys, USERNAME)
+        authorize_sshkeys(keys, login_username())
 
 
 if __name__ == "__main__":
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 6c1bc1e..0ec4846 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -2,6 +2,8 @@
 """Focused checks for the credential-free v19 EBS population path."""
 
 import importlib
+import importlib.machinery
+import importlib.util
 from pathlib import Path
 import subprocess
 import sys
@@ -11,10 +13,18 @@ from unittest import mock
 
 
 EC2_ROOT = Path(__file__).resolve().parents[1] / "bin/ec2"
+SSHKEYS_HOOK = (
+    Path(__file__).resolve().parents[1]
+    / "patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys"
+)
 sys.path.insert(0, str(EC2_ROOT))
 sys.modules.setdefault("ec2metadata", mock.Mock())
 ebs_bundle = importlib.import_module("ebs_bundle")
 ebs_populate = importlib.import_module("ebs_populate")
+sshkeys_loader = importlib.machinery.SourceFileLoader("ec2_sshkeys", str(SSHKEYS_HOOK))
+sshkeys_spec = importlib.util.spec_from_loader("ec2_sshkeys", sshkeys_loader)
+sshkeys = importlib.util.module_from_spec(sshkeys_spec)
+sshkeys_loader.exec_module(sshkeys)
 
 
 class FakeDevice:
@@ -43,6 +53,19 @@ class FakeDevice:
 
 
 class PopulateTests(unittest.TestCase):
+    def test_ssh_key_follows_the_configured_login_principal(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            config = Path(temporary) / "inithooks"
+            for value, expected in (
+                ("true", "admin"), ("TRUE", "admin"), ("false", "root"),
+            ):
+                with self.subTest(value=value):
+                    config.write_text(f"SUDOADMIN={value}\n", encoding="utf-8")
+                    self.assertEqual(expected, sshkeys.login_username(config))
+
+            config.write_text("OTHER=value\n", encoding="utf-8")
+            self.assertEqual("root", sshkeys.login_username(config))
+
     def test_local_command_imports_without_aws_site_packages(self):
         completed = subprocess.run(
             [sys.executable, "-S", str(EC2_ROOT / "ebs_populate.py"), "--help"],

From 796396c0f3f95c345febecf4001d35cbd79adfe1 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 03:57:45 +0800
Subject: [PATCH 07/21] Normalize populated EBS root permissions

Archive-mode rootfs synchronization can copy a private extraction directory mode onto the mounted filesystem root. That prevents non-root EC2 principals from traversing to their authorized_keys file even when the key hook selects the correct account.

Set the populated filesystem root to the conventional 0755 mode before installing GRUB. The focused EC2 test reproduces a 0700 source container and verifies every chroot command observes the normalized mode.
---
 bin/ec2/ebs_bundle.py |  1 +
 tests/ec2-v19         | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/bin/ec2/ebs_bundle.py b/bin/ec2/ebs_bundle.py
index bd8e1a6..0d44036 100755
--- a/bin/ec2/ebs_bundle.py
+++ b/bin/ec2/ebs_bundle.py
@@ -217,6 +217,7 @@ def populate(rootfs, device, filesystem="ext4"):
     try:
         log.info("syncing rootfs to partition")
         utils.rsync(rootfs, mount_path)
+        os.chmod(mount_path, 0o755)
 
         log.info("installing GRUB on volume")
         for submount in ("/sys", "/proc", "/dev"):
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 0ec4846..a74106f 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -123,6 +123,28 @@ class PopulateTests(unittest.TestCase):
             self.assertIn((["chroot", rootfs + ".mount", "grub-install", "/dev/test"], {"check": True}), commands)
             self.assertFalse(Path(rootfs + ".mount").exists())
 
+    def test_populate_normalizes_filesystem_root_before_chroot(self):
+        with tempfile.TemporaryDirectory() as temporary:
+            rootfs = str(Path(temporary) / "rootfs")
+            Path(rootfs).mkdir(mode=0o700)
+            device = FakeDevice()
+            root_modes = []
+
+            def rsync(source, destination):
+                Path(destination).chmod(Path(source).stat().st_mode & 0o777)
+
+            def run(command, **kwargs):
+                if command[:2] == ["chroot", rootfs + ".mount"]:
+                    root_modes.append(Path(rootfs + ".mount").stat().st_mode & 0o777)
+                return mock.Mock(returncode=0)
+
+            with mock.patch.object(ebs_bundle.subprocess, "run", side_effect=run), \
+                    mock.patch.object(ebs_bundle.utils, "rsync", side_effect=rsync):
+                ebs_bundle.populate(rootfs, device)
+
+            self.assertTrue(root_modes)
+            self.assertEqual(root_modes, [0o755] * len(root_modes))
+
     def test_unmounts_after_chroot_failure(self):
         with tempfile.TemporaryDirectory() as temporary:
             rootfs = str(Path(temporary) / "rootfs")

From f74950054f5025d42500259ae717a91e212f8f02 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 12:31:57 +0000
Subject: [PATCH 08/21] Enable automatic TLS for Hub hostnames
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Hub launches already reserve and publish a tklapp.com hostname, but EC2 guests keep serving their generated self-signed certificate until an administrator runs Confconsole manually. Request a Let’s Encrypt certificate after HubDNS publishes the instance address, retain an hourly retry only while DNS or ACME is unavailable, and leave Confconsole’s daily renewal path in charge after success.

Point Webmin at the generated intermediate chain so its TLS endpoint validates alongside appliance HTTPS services. The automation is intentionally limited to TurnKey-managed tklapp.com names; custom domains keep the existing explicit flow.
---
 .../cron.hourly/turnkey-tklapp-letsencrypt    |   1 +
 .../inithooks/firstboot.d/82hub-letsencrypt   |   8 ++
 .../usr/local/sbin/turnkey-tklapp-letsencrypt | 123 ++++++++++++++++++
 3 files changed, 132 insertions(+)
 create mode 120000 patches/ec2/overlay/etc/cron.hourly/turnkey-tklapp-letsencrypt
 create mode 100755 patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt
 create mode 100755 patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt

diff --git a/patches/ec2/overlay/etc/cron.hourly/turnkey-tklapp-letsencrypt b/patches/ec2/overlay/etc/cron.hourly/turnkey-tklapp-letsencrypt
new file mode 120000
index 0000000..ce38557
--- /dev/null
+++ b/patches/ec2/overlay/etc/cron.hourly/turnkey-tklapp-letsencrypt
@@ -0,0 +1 @@
+/usr/local/sbin/turnkey-tklapp-letsencrypt
\ No newline at end of file
diff --git a/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt b/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt
new file mode 100755
index 0000000..4449436
--- /dev/null
+++ b/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if ! /usr/local/sbin/turnkey-tklapp-letsencrypt; then
+    logger -t turnkey-tklapp-letsencrypt \
+        "Initial certificate request failed; the hourly retry remains enabled"
+fi
+
+exit 0
diff --git a/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt b/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
new file mode 100755
index 0000000..a4cb6aa
--- /dev/null
+++ b/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+set -u
+
+readonly APP="${0##*/}"
+readonly RETRY_HOOK="/etc/cron.hourly/turnkey-tklapp-letsencrypt"
+readonly HUBDNS_FQDN="/var/lib/hubdns/fqdn"
+readonly WRAPPER="/usr/lib/confconsole/plugins.d/Lets_Encrypt/dehydrated-wrapper"
+readonly SHARE="/usr/share/confconsole/letsencrypt"
+readonly DEHYDRATED_ETC="/etc/dehydrated"
+readonly CONFIG="$DEHYDRATED_ETC/confconsole.config"
+readonly DOMAINS="$DEHYDRATED_ETC/confconsole.domains.txt"
+readonly CERT="/etc/ssl/private/cert.pem"
+
+log() {
+    logger -t "$APP" -- "$*"
+    echo "$APP: $*"
+}
+
+disable_retry() {
+    rm -f -- "$RETRY_HOOK"
+}
+
+certificate_is_current() {
+    openssl x509 -in "$CERT" -noout \
+        -checkhost "$fqdn" -checkend 2592000 >/dev/null 2>&1
+}
+
+configure_webmin_chain() {
+    local chain
+    local miniserv
+
+    chain="/var/lib/dehydrated/certs/$fqdn/chain.pem"
+    miniserv="/etc/webmin/miniserv.conf"
+    [[ -s "$chain" && -f "$miniserv" ]] || return 0
+
+    if grep -q '^extracas=' "$miniserv"; then
+        sed -i "s|^extracas=.*|extracas=$chain|" "$miniserv"
+    else
+        printf 'extracas=%s\n' "$chain" >> "$miniserv"
+    fi
+    systemctl try-restart webmin.service >/dev/null 2>&1 || true
+}
+
+fqdn="${FQDN:-}"
+if [[ -z "$fqdn" && -s "$HUBDNS_FQDN" ]]; then
+    read -r fqdn < "$HUBDNS_FQDN"
+fi
+fqdn="${fqdn%.}"
+fqdn="${fqdn,,}"
+
+case "$fqdn" in
+    *.tklapp.com) ;;
+    *)
+        disable_retry
+        exit 0
+        ;;
+esac
+
+if [[ ! "$fqdn" =~ ^[a-z0-9.-]+$ ]]; then
+    log "Refusing invalid Hub-managed hostname: $fqdn"
+    disable_retry
+    exit 0
+fi
+
+if [[ ! -x "$WRAPPER" ]]; then
+    log "Confconsole Let's Encrypt client is unavailable"
+    exit 1
+fi
+
+if certificate_is_current; then
+    configure_webmin_chain
+    disable_retry
+    exit 0
+fi
+
+public_ip="$(ec2metadata --public-ipv4 2>/dev/null || true)"
+if [[ ! "$public_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+    log "Public IPv4 address is not available yet"
+    exit 1
+fi
+
+dns_ready=
+for _attempt in {1..24}; do
+    resolved="$(getent ahostsv4 "$fqdn" 2>/dev/null \
+        | awk '{print $1}' | sort -u || true)"
+    if grep -Fxq -- "$public_ip" <<< "$resolved"; then
+        dns_ready=y
+        break
+    fi
+    sleep 5
+done
+
+if [[ -z "$dns_ready" ]]; then
+    log "$fqdn does not resolve to this instance yet; deferring certificate request"
+    exit 1
+fi
+
+install -d -m 0755 "$DEHYDRATED_ETC"
+if [[ ! -f "$CONFIG" ]]; then
+    install -m 0644 "$SHARE/dehydrated-confconsole.config" "$CONFIG"
+fi
+
+domains_tmp="$(mktemp "$DEHYDRATED_ETC/confconsole.domains.txt.XXXXXX")"
+printf '%s\n' "$fqdn" > "$domains_tmp"
+chmod 0644 "$domains_tmp"
+mv -f -- "$domains_tmp" "$DOMAINS"
+
+if ! "$WRAPPER" --register --challenge http-01 --log-info; then
+    log "Let's Encrypt certificate request failed"
+    exit 1
+fi
+
+if ! certificate_is_current; then
+    log "Let's Encrypt did not install a current certificate for $fqdn"
+    exit 1
+fi
+
+configure_webmin_chain
+disable_retry
+log "Installed and enabled automatic renewal for $fqdn"
+
+exit 0

From 2d5ad56ca407abbdfd9b5d0ac31d7b9726f8181f Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 12:46:41 +0000
Subject: [PATCH 09/21] Install the corrected HubDNS v19 candidate

The published HubDNS 1.4.0 package has broken command links and imports the obsolete Python curl-wrapper API, which prevents Hub first boot from assigning its reserved hostname. Verify and install the reviewed 1.4.0+fix1 package while applying the EC2 image layer, then record its artifact digest and source commits in the guest.

Keep the candidate ahead of hubclient installation and assert the installed package version. Extend the focused EC2 checks to cover both the package provenance and the tklapp.com-only certificate automation.
---
 patches/ec2/conf | 19 +++++++++++++++++++
 tests/ec2-v19    | 17 +++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/patches/ec2/conf b/patches/ec2/conf
index 7f35aa7..24aee3b 100755
--- a/patches/ec2/conf
+++ b/patches/ec2/conf
@@ -14,10 +14,17 @@ echo "$tklamq_sha256  $tklamq_candidate" | sha256sum --check --strict
 install "$tklamq_candidate"
 rm -f "$tklamq_candidate"
 
+hubdns_candidate=/root/hubdns_1.4.0+fix1_all.deb
+hubdns_sha256=49d0363dfa628b1c6503ba5d994283318cb120697aafc0bff038095ac34f45fc
+echo "$hubdns_sha256  $hubdns_candidate" | sha256sum --check --strict
+install "$hubdns_candidate"
+rm -f "$hubdns_candidate"
+
 # hubclient remains an exact signed-archive input. The local tklamq version is
 # a normal package version so a later signed archive release can supersede it.
 install hubclient=0.3.0 sudo parted gdisk #ebsmount
 test "$(dpkg-query -W -f='${Version}' tklamq)" = 0.12.1
+test "$(dpkg-query -W -f='${Version}' hubdns)" = 1.4.0+fix1
 
 /usr/bin/install -d -m 0755 /usr/share/doc/tklamq
 cat > /usr/share/doc/tklamq/turnkey-v19-build-input.json <<'EOF'
@@ -29,6 +36,18 @@ cat > /usr/share/doc/tklamq/turnkey-v19-build-input.json <<'EOF'
 }
 EOF
 
+/usr/bin/install -d -m 0755 /usr/share/doc/hubdns
+cat > /usr/share/doc/hubdns/turnkey-v19-build-input.json <<'EOF'
+{
+  "artifact": "hubdns_1.4.0+fix1_all.deb",
+  "sha256": "49d0363dfa628b1c6503ba5d994283318cb120697aafc0bff038095ac34f45fc",
+  "source_commits": [
+    "cc127d63c306089059ab8eebe1d67b1db119cca6",
+    "75b616701c55fb0a2aac2b4e9535f30ed29accfb"
+  ]
+}
+EOF
+
 # grub tweaks
 DEFAULT=/etc/default/grub
 sed -i 's/^\(GRUB_CMDLINE_LINUX_DEFAULT=.*\)"$/\1 xencons=ttyS0 console=ttyS0"/' $DEFAULT
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index a74106f..3be2389 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -17,6 +17,10 @@ SSHKEYS_HOOK = (
     Path(__file__).resolve().parents[1]
     / "patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys"
 )
+TLS_HELPER = (
+    Path(__file__).resolve().parents[1]
+    / "patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt"
+)
 sys.path.insert(0, str(EC2_ROOT))
 sys.modules.setdefault("ec2metadata", mock.Mock())
 ebs_bundle = importlib.import_module("ebs_bundle")
@@ -93,6 +97,19 @@ class PopulateTests(unittest.TestCase):
         for forbidden in ("trusted=yes", "sources.list", "preferences.d", "GH_TOKEN", "AWS_"):
             self.assertNotIn(forbidden, conf)
 
+        hubdns_digest = "49d0363dfa628b1c6503ba5d994283318cb120697aafc0bff038095ac34f45fc"
+        self.assertIn(hubdns_digest, conf)
+        self.assertLess(conf.index('install "$hubdns_candidate"'), hubclient_install)
+        self.assertIn('rm -f "$hubdns_candidate"', conf)
+
+    def test_tls_automation_is_limited_to_hub_managed_names(self):
+        helper = TLS_HELPER.read_text()
+        self.assertIn("*.tklapp.com", helper)
+        self.assertIn("--register --challenge http-01", helper)
+        self.assertIn('-checkhost "$fqdn"', helper)
+        self.assertIn("/chain.pem", helper)
+        self.assertIn("disable_retry", helper)
+
     def test_nvme_partition_path_uses_p_separator(self):
         device = ebs_bundle.Device("/dev/nvme1n1")
         with mock.patch.object(ebs_bundle.subprocess, "run") as run, \

From 154a97290cf697805f4b7248beb047099c2ce61c Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 14:19:30 +0000
Subject: [PATCH 10/21] Keep automatic TLS pending until ACME is ready

The first-boot TLS helper could run before HubDNS assigned a hostname and remove its own hourly retry. It also combined OpenSSL hostname and expiration checks in a way that let the final expiration result mask a hostname mismatch, allowing the self-signed appliance certificate to be mistaken for an ACME certificate.

Keep missing HubDNS names retryable and require the dehydrated full-chain certificate to pass separate hostname and lifetime checks before disabling retries. Explicit non-tklapp.com names still disable the automation.

Verified with shell syntax validation and the focused ec2-v19 test suite.
---
 .../usr/local/sbin/turnkey-tklapp-letsencrypt        | 12 ++++++++++--
 tests/ec2-v19                                        |  7 +++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt b/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
index a4cb6aa..8de7b6b 100755
--- a/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
+++ b/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
@@ -22,8 +22,11 @@ disable_retry() {
 }
 
 certificate_is_current() {
-    openssl x509 -in "$CERT" -noout \
-        -checkhost "$fqdn" -checkend 2592000 >/dev/null 2>&1
+    local managed_cert="/var/lib/dehydrated/certs/$fqdn/fullchain.pem"
+
+    [[ -s "$managed_cert" ]] || return 1
+    openssl x509 -in "$managed_cert" -noout -checkhost "$fqdn" >/dev/null 2>&1 &&
+        openssl x509 -in "$managed_cert" -noout -checkend 2592000 >/dev/null 2>&1
 }
 
 configure_webmin_chain() {
@@ -49,6 +52,11 @@ fi
 fqdn="${fqdn%.}"
 fqdn="${fqdn,,}"
 
+if [[ -z "$fqdn" ]]; then
+    log "Hub-managed hostname is not available yet; deferring certificate request"
+    exit 1
+fi
+
 case "$fqdn" in
     *.tklapp.com) ;;
     *)
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 3be2389..4de3f87 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -107,9 +107,16 @@ class PopulateTests(unittest.TestCase):
         self.assertIn("*.tklapp.com", helper)
         self.assertIn("--register --challenge http-01", helper)
         self.assertIn('-checkhost "$fqdn"', helper)
+        self.assertIn('managed_cert="/var/lib/dehydrated/certs/$fqdn/fullchain.pem"', helper)
+        self.assertEqual(helper.count('openssl x509 -in "$managed_cert" -noout'), 2)
         self.assertIn("/chain.pem", helper)
         self.assertIn("disable_retry", helper)
 
+    def test_tls_automation_retries_until_hubdns_assigns_a_name(self):
+        helper = TLS_HELPER.read_text()
+        self.assertIn('if [[ -z "$fqdn" ]]; then', helper)
+        self.assertIn("hostname is not available yet; deferring certificate request", helper)
+
     def test_nvme_partition_path_uses_p_separator(self):
         device = ebs_bundle.Device("/dev/nvme1n1")
         with mock.patch.object(ebs_bundle.subprocess, "run") as run, \

From 7399233a09f7b6da0014e5720fbf0b39c01680a4 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 15:07:20 +0000
Subject: [PATCH 11/21] Use the persistent HubDNS package in EC2 images

Replace the earlier HubDNS candidate digest with the package containing the
systemd lifecycle fix. The new package keeps the exact tklapp.com record
published after its one-shot update, preventing fresh instances from falling
back to the Hub wildcard endpoint after boot.

Record the source commit in the installed provenance metadata. The focused EC2
contract tests and shell syntax checks pass with the updated candidate.
---
 patches/ec2/conf | 7 ++++---
 tests/ec2-v19    | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/patches/ec2/conf b/patches/ec2/conf
index 24aee3b..a559dba 100755
--- a/patches/ec2/conf
+++ b/patches/ec2/conf
@@ -15,7 +15,7 @@ install "$tklamq_candidate"
 rm -f "$tklamq_candidate"
 
 hubdns_candidate=/root/hubdns_1.4.0+fix1_all.deb
-hubdns_sha256=49d0363dfa628b1c6503ba5d994283318cb120697aafc0bff038095ac34f45fc
+hubdns_sha256=4e255c838cbd6c2a6f64041c3a3d088acd3166d935f60aaf41d30d39d95b6352
 echo "$hubdns_sha256  $hubdns_candidate" | sha256sum --check --strict
 install "$hubdns_candidate"
 rm -f "$hubdns_candidate"
@@ -40,10 +40,11 @@ EOF
 cat > /usr/share/doc/hubdns/turnkey-v19-build-input.json <<'EOF'
 {
   "artifact": "hubdns_1.4.0+fix1_all.deb",
-  "sha256": "49d0363dfa628b1c6503ba5d994283318cb120697aafc0bff038095ac34f45fc",
+  "sha256": "4e255c838cbd6c2a6f64041c3a3d088acd3166d935f60aaf41d30d39d95b6352",
   "source_commits": [
     "cc127d63c306089059ab8eebe1d67b1db119cca6",
-    "75b616701c55fb0a2aac2b4e9535f30ed29accfb"
+    "75b616701c55fb0a2aac2b4e9535f30ed29accfb",
+    "8706ec3a5acc3b3118dbec8dc8f681c6a999160b"
   ]
 }
 EOF
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 4de3f87..12119c0 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -97,7 +97,7 @@ class PopulateTests(unittest.TestCase):
         for forbidden in ("trusted=yes", "sources.list", "preferences.d", "GH_TOKEN", "AWS_"):
             self.assertNotIn(forbidden, conf)
 
-        hubdns_digest = "49d0363dfa628b1c6503ba5d994283318cb120697aafc0bff038095ac34f45fc"
+        hubdns_digest = "4e255c838cbd6c2a6f64041c3a3d088acd3166d935f60aaf41d30d39d95b6352"
         self.assertIn(hubdns_digest, conf)
         self.assertLess(conf.index('install "$hubdns_candidate"'), hubclient_install)
         self.assertIn('rm -f "$hubdns_candidate"', conf)

From afa945d451f91b9d61e478d14b1558910cfedeb8 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 15:31:56 +0000
Subject: [PATCH 12/21] Preserve AWS session tokens during image conversion

Pass the temporary session token alongside the access key and secret when the
Python 3 EC2 client is created. Without it, scoped instance-role credentials
fail authentication before an AMI conversion can create its build volume.

Cover the complete temporary credential tuple in the focused EC2 tests. The
full EC2 contract suite and Python compilation pass.
---
 bin/ec2/utils.py |  3 ++-
 tests/ec2-v19    | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/bin/ec2/utils.py b/bin/ec2/utils.py
index 6545e36..b5c03a8 100644
--- a/bin/ec2/utils.py
+++ b/bin/ec2/utils.py
@@ -25,7 +25,8 @@ def connect_boto3(region=None):
         "ec2",
         region_name=region,
         aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
-        aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"))
+        aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
+        aws_session_token=os.environ.get("AWS_SESSION_TOKEN"))
 
 
 def get_turnkey_version(rootfs):
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 12119c0..ba1cc2b 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -4,6 +4,7 @@
 import importlib
 import importlib.machinery
 import importlib.util
+import os
 from pathlib import Path
 import subprocess
 import sys
@@ -57,6 +58,25 @@ class FakeDevice:
 
 
 class PopulateTests(unittest.TestCase):
+    def test_boto3_client_preserves_temporary_credential_token(self):
+        boto3 = mock.Mock()
+        credentials = {
+            "AWS_ACCESS_KEY_ID": "temporary-access-key",
+            "AWS_SECRET_ACCESS_KEY": "temporary-secret-key",
+            "AWS_SESSION_TOKEN": "temporary-session-token",
+        }
+        with mock.patch.dict(os.environ, credentials, clear=False), \
+                mock.patch.dict(sys.modules, {"boto3": boto3}):
+            ebs_bundle.utils.connect_boto3("us-east-1")
+
+        boto3.client.assert_called_once_with(
+            "ec2",
+            region_name="us-east-1",
+            aws_access_key_id="temporary-access-key",
+            aws_secret_access_key="temporary-secret-key",
+            aws_session_token="temporary-session-token",
+        )
+
     def test_ssh_key_follows_the_configured_login_principal(self):
         with tempfile.TemporaryDirectory() as temporary:
             config = Path(temporary) / "inithooks"

From 4419b264be5472ca8aff1909044ff8e1bcddf665 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 15:40:25 +0000
Subject: [PATCH 13/21] Allow credential-free EBS population

Add an explicit local-device conversion path that prepares a pre-attached block
device while leaving volume creation, snapshots, image registration, and tags
to the release controller. This keeps AWS credentials out of the environment
that extracts and modifies appliance code.

Retain the existing direct AWS path for compatibility. The focused EC2 tests
cover command routing, and the complete EC2 contract suite and shell syntax
checks pass.
---
 bt-ec2        | 11 +++++++++--
 tests/ec2-v19 | 13 +++++++++++++
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/bt-ec2 b/bt-ec2
index 1626fc3..b32e5ae 100755
--- a/bt-ec2
+++ b/bt-ec2
@@ -38,6 +38,8 @@ Options::
                           (implies --secupdates)
     --testing           - allow use of current buildtasks branch (conflicts
                           with --publish &/or --marketplace &/or --copy)
+    --local-device=     - populate this pre-attached block device and leave
+                          AWS snapshot and image registration to the caller
     --pvmshim           - apply paravirtual shim so snapshot is pvm compat.
     --pvmregister       - register pvm-virtualized snapshot, too
 
@@ -49,7 +51,7 @@ EOF
 }
 
 ARGS=("$@")
-unset force secupdates all_updates increment testing pvmshim appver
+unset force secupdates all_updates increment testing local_device pvmshim appver
 ebs_opts=()
 while [[ $# -ne 0 ]]; do
     case $1 in
@@ -62,6 +64,7 @@ while [[ $# -ne 0 ]]; do
         --secupdates)   secupdates="yes";;
         --increment)    increment="yes";;
         --testing)      testing="yes";;
+        --local-device=*) local_device=${1#*=};;
         --pvmshim)      pvmshim="yes";;
         --pvmregister)  ebs_opts+=("$1");;
         *)              if [[ -n "$appver" ]]; then
@@ -245,7 +248,11 @@ umount -l "$rootfs/proc" || true
 "$BT/bin/rootfs-cleanup" "$rootfs"
 "$BT/bin/aptconf-tag" "$rootfs" ec2
 "$BT/bin/build-tag" "$rootfs" ec2
-if [[ -f /usr/bin/python ]]; then
+if [[ -n "$local_device" ]]; then
+    [[ -b "$local_device" ]] \
+        || fatal "local device is not a block device: $local_device"
+    "$BT/bin/ec2/ebs_populate.py" "$rootfs" "$local_device"
+elif [[ -f /usr/bin/python ]]; then
     "$BT/bin/ec2/legacy/ebs.py" "${ebs_opts[@]}" "$rootfs"
 else
     "$BT/bin/ec2/ebs.py" "${ebs_opts[@]}" "$rootfs"
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index ba1cc2b..65355ee 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -14,6 +14,7 @@ from unittest import mock
 
 
 EC2_ROOT = Path(__file__).resolve().parents[1] / "bin/ec2"
+BT_EC2 = Path(__file__).resolve().parents[1] / "bt-ec2"
 SSHKEYS_HOOK = (
     Path(__file__).resolve().parents[1]
     / "patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys"
@@ -58,6 +59,18 @@ class FakeDevice:
 
 
 class PopulateTests(unittest.TestCase):
+    def test_bt_ec2_can_delegate_aws_orchestration_to_its_caller(self):
+        command = BT_EC2.read_text()
+        self.assertIn("--local-device=*", command)
+        self.assertIn('[[ -b "$local_device" ]]', command)
+        local_populate = command.index(
+            '"$BT/bin/ec2/ebs_populate.py" "$rootfs" "$local_device"'
+        )
+        direct_aws = command.index(
+            '"$BT/bin/ec2/ebs.py" "${ebs_opts[@]}" "$rootfs"'
+        )
+        self.assertLess(local_populate, direct_aws)
+
     def test_boto3_client_preserves_temporary_credential_token(self):
         boto3 = mock.Mock()
         credentials = {

From 626f9e8d5af2525d2f1064c52b31c3440383b718 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 15:51:12 +0000
Subject: [PATCH 14/21] Verify v19 images with the Trixie signing key

Replace the Bookworm images fingerprint in the v19 build configuration with
the Trixie primary fingerprint used for the release. This lets conversion and
publication tooling authenticate signed v19 hash files against the intended
release identity.

Add a focused regression assertion that rejects the former fingerprint. The
complete EC2 contract suite passes.
---
 config.example/common.cfg | 2 +-
 tests/ec2-v19             | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/config.example/common.cfg b/config.example/common.cfg
index ae47498..5ea5df4 100644
--- a/config.example/common.cfg
+++ b/config.example/common.cfg
@@ -4,7 +4,7 @@ export BT_PUBLISH_IMGS="s3://${BUCKET}/images"
 export BT_PUBLISH_META="s3://${BUCKET}/metadata"
 export BT_PUBLISH_SCREENS="s3://${BUCKET}/screens"
 export BT_PUBLISH_PROFILES="s3://${BUCKET}/profiles"
-export BT_GPGKEY=26147592087C0EDE42143B637761DEBABBCFBA7C
+export BT_GPGKEY=DEE90D9970AE35D55B7C6C2DC6D254B76A9D9430
 export BT_ISOS=/mnt/isos
 export BT_IMGS=/mnt/imgs
 export BT_QEMU=/mnt/qemu
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 65355ee..20aeb91 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -59,6 +59,13 @@ class FakeDevice:
 
 
 class PopulateTests(unittest.TestCase):
+    def test_v19_iso_verification_uses_the_trixie_images_key(self):
+        config = (EC2_ROOT.parents[1] / "config.example/common.cfg").read_text()
+        trixie_key = "DEE90D9970AE35D55B7C6C2DC6D254B76A9D9430"
+        bookworm_key = "26147592087C0EDE42143B637761DEBABBCFBA7C"
+        self.assertIn(f"export BT_GPGKEY={trixie_key}", config)
+        self.assertNotIn(bookworm_key, config)
+
     def test_bt_ec2_can_delegate_aws_orchestration_to_its_caller(self):
         command = BT_EC2.read_text()
         self.assertIn("--local-device=*", command)

From 3f9cc1fb36bb291fed2f3c52bab6a6dac1e65569 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 16:33:03 +0000
Subject: [PATCH 15/21] Wait for HTTP challenges before ACME validation

The Confconsole HTTP-01 hook returned as soon as it sent a challenge path to the local add-water process. A fast ACME request could arrive before that process registered the token, causing a public 404 and leaving automatic tklapp.com TLS on its hourly retry path.

Probe the exact token through the local HTTP endpoint and compare its content before returning to dehydrated. This preserves the existing challenge server while closing the publication race observed on a Hub-launched v19 WordPress instance.
---
 .../dehydrated-confconsole.hook-http-01.sh    | 77 +++++++++++++++++++
 tests/ec2-v19                                 | 12 +++
 2 files changed, 89 insertions(+)
 create mode 100755 patches/ec2/overlay/usr/share/confconsole/letsencrypt/dehydrated-confconsole.hook-http-01.sh

diff --git a/patches/ec2/overlay/usr/share/confconsole/letsencrypt/dehydrated-confconsole.hook-http-01.sh b/patches/ec2/overlay/usr/share/confconsole/letsencrypt/dehydrated-confconsole.hook-http-01.sh
new file mode 100755
index 0000000..d66dccb
--- /dev/null
+++ b/patches/ec2/overlay/usr/share/confconsole/letsencrypt/dehydrated-confconsole.hook-http-01.sh
@@ -0,0 +1,77 @@
+#!/bin/bash -e
+
+# This dehydrated hook script is packaged with Confconsole.
+# It is designed to be used in conjunction with the TurnKey dehydrated-wrapper.
+# For more info, please see https://www.turnkeylinux.org/docs/letsencypt
+
+# HTTP-01 Hook Script
+
+function hook_log {
+    default="[$(date "+%F %T")] $(basename "$0"):"
+    case ${1} in
+        info)    echo "$default INFO: ${2}";;
+        success) echo "$default SUCCESS: ${2}" >&2;;
+        fatal)   echo "$default FATAL: ${2}" >&2; exit 1;;
+    esac
+}
+
+for var in HTTP HTTP_BIN HTTP_PID HTTP_LOG TKL_KEYFILE TKL_CERTFILE TKL_COMBINED TKL_DHPARAM; do
+    eval "z=\$$var"
+    [[ -z "$z" ]] && hook_log fatal "$var is not set. Exiting..."
+done
+
+function deploy_challenge {
+    local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}"
+    local challenge_url="http://127.0.0.1/.well-known/acme-challenge/$TOKEN_FILENAME"
+    local token_path="$WELLKNOWN/$TOKEN_FILENAME"
+
+    hook_log info "Deploying challenge for $DOMAIN"
+    hook_log info "Serving $token_path on http://$DOMAIN/.well-known/acme-challenge/$TOKEN_FILENAME"
+    $HTTP_BIN --deploy "$token_path"
+
+    for _attempt in {1..50}; do
+        if curl --silent --fail --max-time 1 "$challenge_url" | cmp -s - "$token_path"; then
+            return 0
+        fi
+        sleep 0.1
+    done
+
+    hook_log fatal "HTTP challenge server did not publish $TOKEN_FILENAME"
+}
+
+function clean_challenge {
+    local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}"
+
+    hook_log info "Clean challenge for $DOMAIN"
+    $HTTP_BIN --clean "$WELLKNOWN/$TOKEN_FILENAME"
+}
+
+function deploy_cert {
+    local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}"
+
+    hook_log success "Cert request successful. Writing relevant files for $DOMAIN."
+    hook_log info "fullchain: $FULLCHAINFILE"
+    hook_log info "keyfile: $KEYFILE"
+    cat "$KEYFILE" > "$TKL_KEYFILE"
+    cat "$FULLCHAINFILE" > "$TKL_CERTFILE"
+    cat "$TKL_CERTFILE" "$TKL_KEYFILE" "$TKL_DHPARAM"  > "$TKL_COMBINED"
+    hook_log success "Files written/created for $DOMAIN: $TKL_CERTFILE - $TKL_KEYFILE - $TKL_COMBINED."
+}
+
+function unchanged_cert {
+    local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}"
+
+    hook_log info "cert for $DOMAIN is unchanged - nothing to do"
+}
+
+HANDLER="$1"; shift
+case "$HANDLER" in
+    deploy_challenge)
+        deploy_challenge "$@";;
+    clean_challenge)
+        clean_challenge "$@";;
+    deploy_cert)
+        deploy_cert "$@";;
+    unchanged_cert)
+        unchanged_cert "$@";;
+esac
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 20aeb91..4a5c0dc 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -23,6 +23,11 @@ TLS_HELPER = (
     Path(__file__).resolve().parents[1]
     / "patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt"
 )
+TLS_CHALLENGE_HOOK = (
+    Path(__file__).resolve().parents[1]
+    / "patches/ec2/overlay/usr/share/confconsole/letsencrypt"
+    / "dehydrated-confconsole.hook-http-01.sh"
+)
 sys.path.insert(0, str(EC2_ROOT))
 sys.modules.setdefault("ec2metadata", mock.Mock())
 ebs_bundle = importlib.import_module("ebs_bundle")
@@ -157,6 +162,13 @@ class PopulateTests(unittest.TestCase):
         self.assertIn('if [[ -z "$fqdn" ]]; then', helper)
         self.assertIn("hostname is not available yet; deferring certificate request", helper)
 
+    def test_tls_challenge_waits_until_the_token_is_served(self):
+        hook = TLS_CHALLENGE_HOOK.read_text()
+        self.assertIn("http://127.0.0.1/.well-known/acme-challenge/", hook)
+        self.assertIn("curl --silent --fail --max-time 1", hook)
+        self.assertIn('cmp -s - "$token_path"', hook)
+        self.assertIn("HTTP challenge server did not publish", hook)
+
     def test_nvme_partition_path_uses_p_separator(self):
         device = ebs_bundle.Device("/dev/nvme1n1")
         with mock.patch.object(ebs_bundle.subprocess, "run") as run, \

From 9b068e372e9df57de6090dd5ca3854a6fc5b2d6c Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 17:14:27 +0000
Subject: [PATCH 16/21] Publish HubDNS before requesting TLS

Start the HubDNS unit after Hub credentials have been provisioned and before the first automatic certificate request. On EC2 the unit runs earlier during system startup, before its credential file exists, so it is skipped and the TLS helper otherwise waits against the old wildcard DNS answer.

The focused EC2 suite now verifies the ordering and passes all 15 tests. This keeps the first certificate request on the normal boot path instead of deferring it to the hourly retry.
---
 .../usr/lib/inithooks/firstboot.d/82hub-letsencrypt    |  5 +++++
 tests/ec2-v19                                          | 10 ++++++++++
 2 files changed, 15 insertions(+)

diff --git a/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt b/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt
index 4449436..568c10e 100755
--- a/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt
+++ b/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt
@@ -1,5 +1,10 @@
 #!/bin/bash
 
+if ! systemctl start hubdns.service; then
+    logger -t turnkey-tklapp-letsencrypt \
+        "HubDNS could not publish the managed hostname before TLS setup"
+fi
+
 if ! /usr/local/sbin/turnkey-tklapp-letsencrypt; then
     logger -t turnkey-tklapp-letsencrypt \
         "Initial certificate request failed; the hourly retry remains enabled"
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 4a5c0dc..503d044 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -28,6 +28,10 @@ TLS_CHALLENGE_HOOK = (
     / "patches/ec2/overlay/usr/share/confconsole/letsencrypt"
     / "dehydrated-confconsole.hook-http-01.sh"
 )
+TLS_FIRSTBOOT_HOOK = (
+    Path(__file__).resolve().parents[1]
+    / "patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt"
+)
 sys.path.insert(0, str(EC2_ROOT))
 sys.modules.setdefault("ec2metadata", mock.Mock())
 ebs_bundle = importlib.import_module("ebs_bundle")
@@ -162,6 +166,12 @@ class PopulateTests(unittest.TestCase):
         self.assertIn('if [[ -z "$fqdn" ]]; then', helper)
         self.assertIn("hostname is not available yet; deferring certificate request", helper)
 
+    def test_tls_firstboot_publishes_hubdns_before_requesting_a_certificate(self):
+        hook = TLS_FIRSTBOOT_HOOK.read_text()
+        publish = hook.index("systemctl start hubdns.service")
+        certificate = hook.index("/usr/local/sbin/turnkey-tklapp-letsencrypt")
+        self.assertLess(publish, certificate)
+
     def test_tls_challenge_waits_until_the_token_is_served(self):
         hook = TLS_CHALLENGE_HOOK.read_text()
         self.assertIn("http://127.0.0.1/.well-known/acme-challenge/", hook)

From beee200fdd4aa4d490ac7100ca4a734e8d140bb1 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 17:25:24 +0000
Subject: [PATCH 17/21] Avoid stale DNS during first TLS setup
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Rotate across the instance resolver list while waiting for the Hub-managed hostname. Route 53 can publish the new record just after the first EC2 resolver lookup, leaving that resolver with the previous wildcard answer and delaying first-boot TLS until the retry hook.

The focused EC2 suite passes all 15 tests. The updated helper was also installed on a reproduced Hub launch with a stale primary-resolver answer and obtained a hostname-valid Let’s Encrypt certificate in 11 seconds without an external resolver override.
---
 patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt | 2 +-
 tests/ec2-v19                                                 | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt b/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
index 8de7b6b..95f142d 100755
--- a/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
+++ b/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
@@ -90,7 +90,7 @@ fi
 
 dns_ready=
 for _attempt in {1..24}; do
-    resolved="$(getent ahostsv4 "$fqdn" 2>/dev/null \
+    resolved="$(RES_OPTIONS=rotate getent ahostsv4 "$fqdn" 2>/dev/null \
         | awk '{print $1}' | sort -u || true)"
     if grep -Fxq -- "$public_ip" <<< "$resolved"; then
         dns_ready=y
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 503d044..5377dde 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -165,6 +165,7 @@ class PopulateTests(unittest.TestCase):
         helper = TLS_HELPER.read_text()
         self.assertIn('if [[ -z "$fqdn" ]]; then', helper)
         self.assertIn("hostname is not available yet; deferring certificate request", helper)
+        self.assertIn('RES_OPTIONS=rotate getent ahostsv4 "$fqdn"', helper)
 
     def test_tls_firstboot_publishes_hubdns_before_requesting_a_certificate(self):
         hook = TLS_FIRSTBOOT_HOOK.read_text()

From 9480cb3c1915a31d7bb326805ed1c0db2edd88ab Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 17:38:42 +0000
Subject: [PATCH 18/21] Verify ACME challenges on the public path

Retry the add-water control connection until its socket is ready, then require three stable token responses through the instance public address before asking the certificate authority to validate. A fresh Hub launch reproduced a 404 even after a single localhost readiness response, while the stronger public-path check completed the next certificate request successfully.

The focused EC2 suite passes all 15 tests, and the updated hook obtained and installed a hostname-valid certificate on the reproduced WordPress launch.
---
 .../dehydrated-confconsole.hook-http-01.sh    | 30 +++++++++++++++----
 tests/ec2-v19                                 |  8 +++--
 2 files changed, 30 insertions(+), 8 deletions(-)

diff --git a/patches/ec2/overlay/usr/share/confconsole/letsencrypt/dehydrated-confconsole.hook-http-01.sh b/patches/ec2/overlay/usr/share/confconsole/letsencrypt/dehydrated-confconsole.hook-http-01.sh
index d66dccb..5e98c5c 100755
--- a/patches/ec2/overlay/usr/share/confconsole/letsencrypt/dehydrated-confconsole.hook-http-01.sh
+++ b/patches/ec2/overlay/usr/share/confconsole/letsencrypt/dehydrated-confconsole.hook-http-01.sh
@@ -22,21 +22,41 @@ done
 
 function deploy_challenge {
     local DOMAIN="${1}" TOKEN_FILENAME="${2}" TOKEN_VALUE="${3}"
-    local challenge_url="http://127.0.0.1/.well-known/acme-challenge/$TOKEN_FILENAME"
+    local challenge_url="http://$DOMAIN/.well-known/acme-challenge/$TOKEN_FILENAME"
+    local public_ip
+    local ready_count=0
     local token_path="$WELLKNOWN/$TOKEN_FILENAME"
 
     hook_log info "Deploying challenge for $DOMAIN"
     hook_log info "Serving $token_path on http://$DOMAIN/.well-known/acme-challenge/$TOKEN_FILENAME"
-    $HTTP_BIN --deploy "$token_path"
+
+    public_ip="$(ec2metadata --public-ipv4 2>/dev/null || true)"
+    if [[ ! "$public_ip" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+        hook_log fatal "Public IPv4 address is unavailable for challenge verification"
+    fi
 
     for _attempt in {1..50}; do
-        if curl --silent --fail --max-time 1 "$challenge_url" | cmp -s - "$token_path"; then
-            return 0
+        if "$HTTP_BIN" --deploy "$token_path" 2>/dev/null; then
+            break
         fi
         sleep 0.1
     done
 
-    hook_log fatal "HTTP challenge server did not publish $TOKEN_FILENAME"
+    for _attempt in {1..50}; do
+        if curl --silent --fail --max-time 1 --noproxy '*' \
+                --resolve "$DOMAIN:80:$public_ip" "$challenge_url" \
+                | cmp -s - "$token_path"; then
+            ((ready_count += 1))
+            if (( ready_count >= 3 )); then
+                return 0
+            fi
+        else
+            ready_count=0
+        fi
+        sleep 0.2
+    done
+
+    hook_log fatal "HTTP challenge was not stable on the public instance address"
 }
 
 function clean_challenge {
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 5377dde..b2779c5 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -175,10 +175,12 @@ class PopulateTests(unittest.TestCase):
 
     def test_tls_challenge_waits_until_the_token_is_served(self):
         hook = TLS_CHALLENGE_HOOK.read_text()
-        self.assertIn("http://127.0.0.1/.well-known/acme-challenge/", hook)
-        self.assertIn("curl --silent --fail --max-time 1", hook)
+        self.assertIn('ec2metadata --public-ipv4', hook)
+        self.assertIn('--resolve "$DOMAIN:80:$public_ip"', hook)
+        self.assertIn("curl --silent --fail --max-time 1 --noproxy '*'", hook)
         self.assertIn('cmp -s - "$token_path"', hook)
-        self.assertIn("HTTP challenge server did not publish", hook)
+        self.assertIn("ready_count >= 3", hook)
+        self.assertIn("HTTP challenge was not stable", hook)
 
     def test_nvme_partition_path_uses_p_separator(self):
         device = ebs_bundle.Device("/dev/nvme1n1")

From 5443e98d1087844c2330b4e88fd678356562b217 Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 17:58:54 +0000
Subject: [PATCH 19/21] Retry initial TLS every minute

A newly launched EC2 instance may not be able to reach its own public address during the first ACME attempt even after HubDNS is published. The existing hourly retry leaves every TLS interface on the self-signed certificate for too long.

Add a non-overlapping minutely retry that removes itself as soon as a current hostname-valid certificate is installed. Keep the first attempt bounded so this transient network condition does not delay first-boot completion. Verified by the 16 focused EC2 v19 tests and shell syntax checks.
---
 .../overlay/etc/cron.d/turnkey-tklapp-letsencrypt   |  2 ++
 .../usr/local/sbin/turnkey-tklapp-letsencrypt       |  3 ++-
 tests/ec2-v19                                       | 13 +++++++++++++
 3 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 patches/ec2/overlay/etc/cron.d/turnkey-tklapp-letsencrypt

diff --git a/patches/ec2/overlay/etc/cron.d/turnkey-tklapp-letsencrypt b/patches/ec2/overlay/etc/cron.d/turnkey-tklapp-letsencrypt
new file mode 100644
index 0000000..838b149
--- /dev/null
+++ b/patches/ec2/overlay/etc/cron.d/turnkey-tklapp-letsencrypt
@@ -0,0 +1,2 @@
+# Retry initial Hub-managed TLS without delaying first-boot completion.
+* * * * * root /usr/bin/flock -n /run/turnkey-tklapp-letsencrypt.lock /usr/local/sbin/turnkey-tklapp-letsencrypt >/dev/null 2>&1
diff --git a/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt b/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
index 95f142d..b170c4a 100755
--- a/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
+++ b/patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt
@@ -4,6 +4,7 @@ set -u
 
 readonly APP="${0##*/}"
 readonly RETRY_HOOK="/etc/cron.hourly/turnkey-tklapp-letsencrypt"
+readonly RETRY_CRON="/etc/cron.d/turnkey-tklapp-letsencrypt"
 readonly HUBDNS_FQDN="/var/lib/hubdns/fqdn"
 readonly WRAPPER="/usr/lib/confconsole/plugins.d/Lets_Encrypt/dehydrated-wrapper"
 readonly SHARE="/usr/share/confconsole/letsencrypt"
@@ -18,7 +19,7 @@ log() {
 }
 
 disable_retry() {
-    rm -f -- "$RETRY_HOOK"
+    rm -f -- "$RETRY_HOOK" "$RETRY_CRON"
 }
 
 certificate_is_current() {
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index b2779c5..4a49eb0 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -32,6 +32,10 @@ TLS_FIRSTBOOT_HOOK = (
     Path(__file__).resolve().parents[1]
     / "patches/ec2/overlay/usr/lib/inithooks/firstboot.d/82hub-letsencrypt"
 )
+TLS_RETRY_CRON = (
+    Path(__file__).resolve().parents[1]
+    / "patches/ec2/overlay/etc/cron.d/turnkey-tklapp-letsencrypt"
+)
 sys.path.insert(0, str(EC2_ROOT))
 sys.modules.setdefault("ec2metadata", mock.Mock())
 ebs_bundle = importlib.import_module("ebs_bundle")
@@ -167,6 +171,15 @@ class PopulateTests(unittest.TestCase):
         self.assertIn("hostname is not available yet; deferring certificate request", helper)
         self.assertIn('RES_OPTIONS=rotate getent ahostsv4 "$fqdn"', helper)
 
+    def test_tls_automation_retries_minutely_until_certificate_is_ready(self):
+        helper = TLS_HELPER.read_text()
+        retry = TLS_RETRY_CRON.read_text()
+        self.assertIn("* * * * * root", retry)
+        self.assertIn("/usr/local/sbin/turnkey-tklapp-letsencrypt", retry)
+        self.assertIn("/usr/bin/flock -n /run/turnkey-tklapp-letsencrypt.lock", retry)
+        self.assertIn('RETRY_CRON="/etc/cron.d/turnkey-tklapp-letsencrypt"', helper)
+        self.assertIn('rm -f -- "$RETRY_HOOK" "$RETRY_CRON"', helper)
+
     def test_tls_firstboot_publishes_hubdns_before_requesting_a_certificate(self):
         hook = TLS_FIRSTBOOT_HOOK.read_text()
         publish = hook.index("systemctl start hubdns.service")

From 797f2067a8e46e3be2e25e133225457fcfc797cb Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Wed, 2 Sep 2026 19:40:26 +0000
Subject: [PATCH 20/21] Enforce key-only admin access on EC2 images

AWS Marketplace prohibits SSH password authentication, root remote login, embedded passwords, and authorized keys. Keep sudoadmin active for Hub and standalone boots, replace generated user hashes with a locked non-secret marker, unexpire admin, and install an EC2-only sshd policy.

Make EC2 conversion validate account state, embedded keys, sudo access, and the effective sshd configuration. Focused tests prevent reintroducing the Hub root-switch hook or weakening the policy.
---
 patches/ec2/conf                              | 39 ++++++++++++++++++-
 .../00-turnkey-aws-marketplace.conf           |  4 ++
 .../lib/inithooks/firstboot.d/28ec2-sudoadmin | 12 ------
 tests/ec2-v19                                 | 38 ++++++++++++++++++
 4 files changed, 79 insertions(+), 14 deletions(-)
 create mode 100644 patches/ec2/overlay/etc/ssh/sshd_config.d/00-turnkey-aws-marketplace.conf
 delete mode 100755 patches/ec2/overlay/usr/lib/inithooks/firstboot.d/28ec2-sudoadmin

diff --git a/patches/ec2/conf b/patches/ec2/conf
index a559dba..4d5122a 100755
--- a/patches/ec2/conf
+++ b/patches/ec2/conf
@@ -62,15 +62,50 @@ chmod 644 /var/lib/inithooks/turnkey-init-fence/htdocs/*.png
 # enable sudoadmin in rootfs
 turnkey-sudoadmin on --disable-setpass
 
-# provide for AWSMP requirment of "no default passwords"
+# AWS Marketplace images authenticate the privileged admin account with the
+# EC2 key injected at first boot. Remove generated password hashes without
+# expiring admin, so key login remains available while passwords stay locked.
 additional_users=$(grep ":[1-9][0-9][0-9][0-9]:" /etc/passwd | cut -d: -f1)
 for user in $additional_users; do
-    passwd -l $user
+    usermod --password '*' "$user"
     if [[ $user = "ansible" ]]; then
         ansible_init=/usr/lib/inithooks/firstboot.d/50ansible-key
         sed -i "/.ssh/ s|root|home/admin|" $ansible_init
     fi
 done
+usermod --expiredate -1 admin
+
+# Load the EC2-only policy before any package defaults. OpenSSH uses the first
+# value it obtains for these global options.
+sshd_config=/etc/ssh/sshd_config
+sshd_include='Include /etc/ssh/sshd_config.d/*.conf'
+sed -i '\|^[[:space:]]*Include[[:space:]]\+/etc/ssh/sshd_config.d/\*\.conf[[:space:]]*$|d' \
+    "$sshd_config"
+sed -i "1i$sshd_include" "$sshd_config"
+
+# Fail the conversion if account state, embedded keys, or the effective SSH
+# policy diverges from the Marketplace guest contract.
+admin_shadow=$(getent shadow admin)
+test "$(echo "$admin_shadow" | cut -d: -f2)" = '*'
+test -z "$(echo "$admin_shadow" | cut -d: -f8)"
+test ! -s /root/.ssh/authorized_keys
+test ! -s /home/admin/.ssh/authorized_keys
+grep -qx 'SUDOADMIN=true' /etc/default/inithooks
+grep -qx 'admin ALL=(ALL) NOPASSWD:ALL' /etc/sudoers.d/99_admin
+
+sshd_testdir=$(mktemp -d)
+trap 'rm -rf "$sshd_testdir"' EXIT
+ssh-keygen -q -t ed25519 -N '' -f "$sshd_testdir/host_key"
+sshd -T -h "$sshd_testdir/host_key" > "$sshd_testdir/effective"
+for required in \
+    'passwordauthentication no' \
+    'kbdinteractiveauthentication no' \
+    'permitrootlogin no' \
+    'pubkeyauthentication yes'; do
+    grep -qx "$required" "$sshd_testdir/effective"
+done
+rm -rf "$sshd_testdir"
+trap - EXIT
 
 # init fence now launches confconsole with full options, so autolaunch disabled
 # on other headless builds. However, we want it on Hub builds.
diff --git a/patches/ec2/overlay/etc/ssh/sshd_config.d/00-turnkey-aws-marketplace.conf b/patches/ec2/overlay/etc/ssh/sshd_config.d/00-turnkey-aws-marketplace.conf
new file mode 100644
index 0000000..8a1bb6e
--- /dev/null
+++ b/patches/ec2/overlay/etc/ssh/sshd_config.d/00-turnkey-aws-marketplace.conf
@@ -0,0 +1,4 @@
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+PermitRootLogin no
+PubkeyAuthentication yes
diff --git a/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/28ec2-sudoadmin b/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/28ec2-sudoadmin
deleted file mode 100755
index a5b3da0..0000000
--- a/patches/ec2/overlay/usr/lib/inithooks/firstboot.d/28ec2-sudoadmin
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash -e
-
-[ -n "$_TURNKEY_INIT" ] && exit 0
-
-if grep -q SERVERID= /var/lib/hubclient/server.conf >/dev/null 2>&1; then
-    # hub launch, disable sudoadmin
-    sed -i "s/^SUDOADMIN=.*/SUDOADMIN=false/" /etc/default/inithooks
-else
-    # non-hub launch
-    exit 0
-fi
-
diff --git a/tests/ec2-v19 b/tests/ec2-v19
index 4a49eb0..c604dd6 100755
--- a/tests/ec2-v19
+++ b/tests/ec2-v19
@@ -19,6 +19,16 @@ SSHKEYS_HOOK = (
     Path(__file__).resolve().parents[1]
     / "patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys"
 )
+EC2_PATCH_CONF = Path(__file__).resolve().parents[1] / "patches/ec2/conf"
+MARKETPLACE_SSH_POLICY = (
+    Path(__file__).resolve().parents[1]
+    / "patches/ec2/overlay/etc/ssh/sshd_config.d"
+    / "00-turnkey-aws-marketplace.conf"
+)
+LEGACY_HUB_ROOT_HOOK = (
+    Path(__file__).resolve().parents[1]
+    / "patches/ec2/overlay/usr/lib/inithooks/firstboot.d/28ec2-sudoadmin"
+)
 TLS_HELPER = (
     Path(__file__).resolve().parents[1]
     / "patches/ec2/overlay/usr/local/sbin/turnkey-tklapp-letsencrypt"
@@ -123,6 +133,34 @@ class PopulateTests(unittest.TestCase):
             config.write_text("OTHER=value\n", encoding="utf-8")
             self.assertEqual("root", sshkeys.login_username(config))
 
+    def test_marketplace_ssh_policy_is_key_only_and_non_root(self):
+        directives = {}
+        for line in MARKETPLACE_SSH_POLICY.read_text().splitlines():
+            key, value = line.split(None, 1)
+            directives[key] = value
+
+        self.assertEqual("no", directives["PasswordAuthentication"])
+        self.assertEqual("no", directives["KbdInteractiveAuthentication"])
+        self.assertEqual("no", directives["PermitRootLogin"])
+        self.assertEqual("yes", directives["PubkeyAuthentication"])
+
+    def test_ec2_conversion_keeps_admin_usable_without_a_password_secret(self):
+        conf = EC2_PATCH_CONF.read_text()
+        self.assertIn("turnkey-sudoadmin on --disable-setpass", conf)
+        self.assertIn("usermod --password '*' \"$user\"", conf)
+        self.assertIn("usermod --expiredate -1 admin", conf)
+        self.assertNotIn("passwd -l", conf)
+        self.assertIn("test ! -s /root/.ssh/authorized_keys", conf)
+        self.assertIn("test ! -s /home/admin/.ssh/authorized_keys", conf)
+        self.assertIn("grep -qx 'SUDOADMIN=true' /etc/default/inithooks", conf)
+        self.assertIn("grep -qx 'admin ALL=(ALL) NOPASSWD:ALL'", conf)
+        self.assertIn('sshd -T -h "$sshd_testdir/host_key"', conf)
+        self.assertIn("'passwordauthentication no'", conf)
+        self.assertIn("'permitrootlogin no'", conf)
+
+    def test_hub_launch_does_not_switch_the_ec2_key_back_to_root(self):
+        self.assertFalse(LEGACY_HUB_ROOT_HOOK.exists())
+
     def test_local_command_imports_without_aws_site_packages(self):
         completed = subprocess.run(
             [sys.executable, "-S", str(EC2_ROOT / "ebs_populate.py"), "--help"],

From 01bf3764b5293c150016e54a32392d52ad773a9c Mon Sep 17 00:00:00 2001
From: Liraz Siri <liraz@liraz.org>
Date: Thu, 3 Sep 2026 04:41:22 +0800
Subject: [PATCH 21/21] Match purge help options exactly

The previous substring check interpreted appliance paths containing -h, including Faveo Helpdesk, as help requests and aborted EC2 conversion before package cleanup. Parse each argument for the exact -h and --help options so ordinary rootfs paths continue through conversion while the documented help flags retain their behavior.

Verified by reproducing the old Faveo path failure, exercising the corrected path with a stub chroot helper, checking both help flags, and running bash syntax and diff checks.
---
 bin/purge-pkgs | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/bin/purge-pkgs b/bin/purge-pkgs
index ffc9049..e96451a 100755
--- a/bin/purge-pkgs
+++ b/bin/purge-pkgs
@@ -43,10 +43,12 @@ EOF
     exit 1
 }
 
-# catch --help first
-if [[ " $* " == *"--help"* ]] || [[ " $* " == *"-h"* ]]; then
-    usage
-fi
+# Catch help options without treating a rootfs path containing "-h" as one.
+for arg in "$@"; do
+    case "$arg" in
+        --help|-h) usage;;
+    esac
+done
 path_to_rootfs=$1
 shift