Repair v19 EBS conversion, Hub TLS, and TKLBAM runtime

bin/ec2/ebs_bundle.py

@@ -30,8 +30,6 @@
 import sys
 import time
 
-from botocore.exceptions import ClientError
-
 import utils
 
 log = utils.get_logger("ebs-bundle")
@@ -102,6 +100,8 @@ def create(self, size, zone=None):
 
     def delete(self, max_attempts=10):
         if self.vol:
+            from botocore.exceptions import ClientError
+
             attempt = 0
             while True:
                 attempt += 1
@@ -153,8 +153,8 @@ def __del__(self):
 
 
 class Device:
-    def __init__(self):
-        self.real_path = self._get_freedevice()
+    def __init__(self, real_path=None):
+        self.real_path = real_path if real_path else self._get_freedevice()
         if not self.real_path:
             raise EbsBundleError("no free devices available...")
 
@@ -197,13 +197,47 @@ def mkpart(self):
         subprocess.run(["partprobe", self.real_path], check=True)
         time.sleep(5)
         self.root_path = self.real_path
-        self.real_path = self.real_path + "2"
+        separator = "p" if os.path.basename(self.real_path).startswith("nvme") else ""
+        self.real_path = self.real_path + separator + "2"
 
     def __del__(self):
         if self.is_mounted():
             self.umount()
 
 
+def populate(rootfs, device, filesystem="ext4"):
+    """Populate one pre-attached block device without making an AWS call."""
+    log.info("creating partitions")
+    device.mkpart()
+    device.mkfs(filesystem)
+    mount_path = rootfs + ".mount"
+    device.mount(mount_path)
+
+    submounts = []
+    try:
+        log.info("syncing rootfs to partition")
+        utils.rsync(rootfs, mount_path)
+        os.chmod(mount_path, 0o755)
+
+        log.info("installing GRUB on volume")
+        for submount in ("/sys", "/proc", "/dev"):
+            subprocess.run(
+                ["mount", "--bind", "--make-rslave", submount, mount_path + submount],
+                check=True,
+            )
+            submounts.append(submount)
+        subprocess.run(["chroot", mount_path, "grub-install", device.root_path], check=True)
+        subprocess.run(["chroot", mount_path, "update-grub"], check=True)
+        subprocess.run(["chroot", mount_path, "update-initramfs", "-u"], check=True)
+    finally:
+        for submount in reversed(submounts):
+            subprocess.run(["umount", "-l", mount_path + submount], check=False)
+        if device.is_mounted():
+            device.umount()
+        if os.path.isdir(mount_path):
+            os.rmdir(mount_path)
+
+
 def bundle(rootfs, snapshot_name, size=10, filesystem="ext4"):
     log.info(f"target snapshot - {snapshot_name} ")
 
@@ -214,33 +248,8 @@ def bundle(rootfs, snapshot_name, size=10, filesystem="ext4"):
     device = Device()
     volume.attach(utils.get_instanceid(), device)
 
-    log.info("creating partitions")
-    device.mkpart()
-    device.mkfs(filesystem)
-    mount_path = rootfs + ".mount"
-    device.mount(mount_path)
-
-    log.info("syncing rootfs to partition")
-    utils.rsync(rootfs, mount_path)
-
-    log.info("installing GRUB on volume")
-    submounts = ["/sys", "/proc", "/dev"]
-    for s in submounts:
-        subprocess.run(["mount", "--bind", "--make-rslave", s, mount_path + s],
-                       check=True)
-    subprocess.run(["chroot", mount_path, "grub-install", device.root_path],
-                   check=True)
-    subprocess.run(["chroot", mount_path, "update-grub"], check=True)
-    subprocess.run(["chroot", mount_path, "update-initramfs", "-u"],
-                   check=True)
-
-    submounts.reverse()
-    for s in submounts:
-        subprocess.run(["umount", "-l", mount_path + s], check=True)
-
-    device.umount()
+    populate(rootfs, device, filesystem)
     volume.detach()
-    os.removedirs(mount_path)
 
     log.info("creating snapshot from volume")
     snapshot = Snapshot()

bin/ec2/ebs_populate.py

@@ -0,0 +1,23 @@
+#!/usr/bin/python3
+"""Populate a pre-attached EBS device from a prepared root filesystem."""
+
+import argparse
+import os
+
+from ebs_bundle import Device, populate
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("rootfs")
+    parser.add_argument("device")
+    args = parser.parse_args()
+    if not os.path.isdir(args.rootfs):
+        parser.error("rootfs path does not exist")
+    if not os.path.exists(args.device):
+        parser.error("device path does not exist")
+    populate(args.rootfs, Device(args.device))
+
+
+if __name__ == "__main__":
+    main()

bin/ec2/utils.py

@@ -14,19 +14,19 @@
 import subprocess
 import sys
 
-import boto3
-import ec2metadata
-
 import conf
 
 
 def connect_boto3(region=None):
+    import boto3
+
     region = region if region else get_region()
     return boto3.client(
         "ec2",
         region_name=region,
         aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
-        aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"))
+        aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
+        aws_session_token=os.environ.get("AWS_SESSION_TOKEN"))
 
 
 def get_turnkey_version(rootfs):
@@ -35,14 +35,20 @@ def get_turnkey_version(rootfs):
 
 
 def get_instanceid():
+    import ec2metadata
+
     return ec2metadata.get("instance-id")
 
 
 def get_zone():
+    import ec2metadata
+
     return ec2metadata.get("availability-zone")
 
 
 def get_region():
+    import ec2metadata
+
     return ec2metadata.get("availability-zone")[0:-1]
 
 

bin/iso-release

@@ -90,7 +90,49 @@ cp build/product.iso "$O/$name.iso"
 if [[ -e $BT_PROFILES/$appname ]]; then
     mkdir -p "$O/$name.tklbam"
     export PROFILES_CONF=$BT_PROFILES
-    "$BT/bin/generate-tklbam-profile" "$O/$name.iso" "$O/$name.tklbam"
+    if [[ -x /usr/lib/tklbam-pypy2/bin/pypy ]]; then
+        "$BT/bin/generate-tklbam-profile" "$O/$name.iso" "$O/$name.tklbam"
+    else
+        (
+            rootfs=$(readlink -f "$rootfs")
+            [[ -x "$rootfs/usr/lib/tklbam-pypy2/bin/pypy" ]] || \
+                fatal "TKLBAM runtime not found in $rootfs"
+            staging=$(mktemp -d "$rootfs/var/tmp/buildtasks-tklbam-profile.XXXXXX")
+            guest_staging=${staging#"$rootfs"}
+            cleanup() {
+                status=$?
+                trap - EXIT
+                rm -rf -- "$staging"
+                exit "$status"
+            }
+            trap cleanup EXIT
+
+            install -d "$staging/profiles" "$staging/output"
+            install -m 0755 "$BT/bin/generate-tklbam-profile" \
+                "$staging/generate-tklbam-profile"
+            cat > "$staging/run-tklbam-profile" <<EOF
+#!/bin/bash
+exec /usr/lib/tklbam-pypy2/bin/pypy "$guest_staging/generate-tklbam-profile" / "$guest_staging/output"
+EOF
+            chmod 0755 "$staging/run-tklbam-profile"
+            cp "$BT_PROFILES/core" "$staging/profiles/core"
+            if [[ "$appname" != "core" ]]; then
+                cp "$BT_PROFILES/$appname" "$staging/profiles/$appname"
+            fi
+            if [[ -d "$BT_PROFILES/hooks/$appname" ]]; then
+                install -d "$staging/profiles/hooks"
+                cp -a "$BT_PROFILES/hooks/$appname" "$staging/profiles/hooks/"
+            fi
+
+            PROFILES_CONF="$guest_staging/profiles" \
+            TKLBAM_LIB_PATH=/usr/lib/tklbam \
+            LD_LIBRARY_PATH=/usr/lib/tklbam-pypy2/bin \
+                fab-chroot \
+                -e PROFILES_CONF:TKLBAM_LIB_PATH:LD_LIBRARY_PATH \
+                "$rootfs" --script "$staging/run-tklbam-profile"
+            cp -a "$staging/output/." "$O/$name.tklbam/"
+        )
+    fi
 fi
 if [[ -z "$no_screens" ]]; then
     mkdir -p "$O/$name.screens"

bin/purge-pkgs

@@ -43,10 +43,12 @@ EOF
     exit 1
 }
 
-# catch --help first
-if [[ " $* " == *"--help"* ]] || [[ " $* " == *"-h"* ]]; then
-    usage
-fi
+# Catch help options without treating a rootfs path containing "-h" as one.
+for arg in "$@"; do
+    case "$arg" in
+        --help|-h) usage;;
+    esac
+done
 path_to_rootfs=$1
 shift
 

bt-ec2

@@ -38,6 +38,8 @@ Options::
                           (implies --secupdates)
     --testing           - allow use of current buildtasks branch (conflicts
                           with --publish &/or --marketplace &/or --copy)
+    --local-device=     - populate this pre-attached block device and leave
+                          AWS snapshot and image registration to the caller
     --pvmshim           - apply paravirtual shim so snapshot is pvm compat.
     --pvmregister       - register pvm-virtualized snapshot, too
 
@@ -49,7 +51,7 @@ EOF
 }
 
 ARGS=("$@")
-unset force secupdates all_updates increment testing pvmshim appver
+unset force secupdates all_updates increment testing local_device pvmshim appver
 ebs_opts=()
 while [[ $# -ne 0 ]]; do
     case $1 in
@@ -62,6 +64,7 @@ while [[ $# -ne 0 ]]; do
         --secupdates)   secupdates="yes";;
         --increment)    increment="yes";;
         --testing)      testing="yes";;
+        --local-device=*) local_device=${1#*=};;
         --pvmshim)      pvmshim="yes";;
         --pvmregister)  ebs_opts+=("$1");;
         *)              if [[ -n "$appver" ]]; then
@@ -245,7 +248,11 @@ umount -l "$rootfs/proc" || true
 "$BT/bin/rootfs-cleanup" "$rootfs"
 "$BT/bin/aptconf-tag" "$rootfs" ec2
 "$BT/bin/build-tag" "$rootfs" ec2
-if [[ -f /usr/bin/python ]]; then
+if [[ -n "$local_device" ]]; then
+    [[ -b "$local_device" ]] \
+        || fatal "local device is not a block device: $local_device"
+    "$BT/bin/ec2/ebs_populate.py" "$rootfs" "$local_device"
+elif [[ -f /usr/bin/python ]]; then
     "$BT/bin/ec2/legacy/ebs.py" "${ebs_opts[@]}" "$rootfs"
 else
     "$BT/bin/ec2/ebs.py" "${ebs_opts[@]}" "$rootfs"

config.example/common.cfg

@@ -4,7 +4,7 @@ export BT_PUBLISH_IMGS="s3://${BUCKET}/images"
 export BT_PUBLISH_META="s3://${BUCKET}/metadata"
 export BT_PUBLISH_SCREENS="s3://${BUCKET}/screens"
 export BT_PUBLISH_PROFILES="s3://${BUCKET}/profiles"
-export BT_GPGKEY=26147592087C0EDE42143B637761DEBABBCFBA7C
+export BT_GPGKEY=DEE90D9970AE35D55B7C6C2DC6D254B76A9D9430
 export BT_ISOS=/mnt/isos
 export BT_IMGS=/mnt/imgs
 export BT_QEMU=/mnt/qemu

patches/cloud/overlay/usr/lib/inithooks/firstboot.d/40ec2-sshkeys

@@ -10,7 +10,17 @@ if '_TURNKEY_INIT' in os.environ:
 import pwd
 import ec2metadata
 
-USERNAME = 'root'
+
+def login_username(config='/etc/default/inithooks'):
+    username = 'root'
+    with open(config) as fob:
+        for line in fob:
+            key, separator, value = line.partition('=')
+            if separator and key.strip() == 'SUDOADMIN':
+                if value.strip().strip("'\"").lower() == 'true':
+                    username = 'admin'
+                break
+    return username
 
 
 def authorize_sshkeys(keys, username):
@@ -31,7 +41,7 @@ def authorize_sshkeys(keys, username):
 def main():
     keys = ec2metadata.get('public-keys')
     if keys:
-        authorize_sshkeys(keys, USERNAME)
+        authorize_sshkeys(keys, login_username())
 
 
 if __name__ == "__main__":