Shared VM helpers: disk import, releases, checksums, first boot, IP by MAC

.github/workflows/verify-split-files.yml

@@ -24,6 +24,7 @@ on:
       - "incus/**"
       - "tests/vm-iso-storage.sh"
       - "tests/vm-consoles.sh"
+      - "tests/vm-helpers.sh"
       - ".github/workflows/verify-split-files.yml"
   push:
     branches: [main]
@@ -39,6 +40,7 @@ on:
       - "incus/**"
       - "tests/vm-iso-storage.sh"
       - "tests/vm-consoles.sh"
+      - "tests/vm-helpers.sh"
       - ".github/workflows/verify-split-files.yml"
   workflow_dispatch:
 
@@ -66,6 +68,9 @@ jobs:
       - name: VM console regression tests
         run: bash tests/vm-consoles.sh
 
+      - name: VM helper regression tests
+        run: bash tests/vm-helpers.sh
+
       - name: Prefetch lists cover every engine file
         run: |
           set -euo pipefail

core/build.func

@@ -236,7 +236,7 @@ _CS_ENGINE_FILES=(
   api/errorlog.func api/exitcodes.func api/sysinfo.func
   api/telemetry.func
   ui/build-ui.func ui/advanced.func ui/defaults.func
-  ui/menu.func ui/validate.func vm/cloud-init.func
+  ui/menu.func ui/validate.func vm/cloud-init.func vm/common.func
   core/core.func core/dev-mode.func core/error_handler.func
   lxc/install.func lxc/platform.func host/preflight.func
   host/source-origin.func lib/tools.func lib/db.func

docs/README.md

@@ -104,6 +104,7 @@ an update before it starts.
 | [`api/sysinfo.func`](../api/sysinfo.func) | Host and container facts, origin detection |
 | [`api/telemetry.func`](../api/telemetry.func) | Payload assembly and delivery |
 | [`vm/cloud-init.func`](../vm/cloud-init.func) | Cloud-init configuration for VMs |
+| [`vm/common.func`](../vm/common.func) | Backend-independent VM script helpers (tools, releases, checksums, first boot, summary) |
 | [`pve/backend.func`](../pve/backend.func) | Container creation via `pct` |
 | [`pve/vm-core.func`](../pve/vm-core.func) | VM creation via `qm` |
 | [`pve/vm-app.func`](../pve/vm-app.func) | Running LXC applications inside a VM |

docs/backends.md

@@ -140,7 +140,8 @@ is changed. Missing or invalid ISO storage is reported before VM creation.
 On Incus this helper explicitly fails: these installer scripts require
 Proxmox's `pvesm` and `qm`.
 
-Run the host-independent regression tests with `bash tests/vm-iso-storage.sh`.
+Run the host-independent regression tests with `bash tests/vm-iso-storage.sh`
+and, for the shared VM helpers, `bash tests/vm-helpers.sh`.
 
 ### Backend-specific VM creation
 

docs/vm.md

@@ -1,4 +1,4 @@
-# `vm/` — cloud-init
+# `vm/` — cloud-init and shared VM helpers
 
 Back to the [index](README.md).
 
@@ -27,8 +27,56 @@ helpers as unavailable is what keeps it on its fallback branch instead of
 letting it call into a function that would need `qm`. `vm_provision` applies
 the same `CLOUDINIT_*` variables through the `cloud-init.*` instance keys.
 
+`get_vm_ip <vmid> [timeout] [mac]` reads addresses through the guest agent.
+Pass the VM's MAC: only that NIC then counts, which is the only reliable answer
+once the guest runs Docker, Podman or Kubernetes. Without a MAC, bridge and
+tunnel interfaces those create are skipped; loopback and link-local addresses
+never count. `wait_for_cloud_init` asks the guest agent
+(`cloud-init status --wait`) first, which needs neither an address nor SSH
+keys, and falls back to SSH only for guests without an agent.
+
 One thing to know before editing it: this file declares its functions as
 `function name() {` rather than `name() {` — the only one that does, apart from
 `get_lxc_ip` in [`core/core.func`](../core/core.func). Anything that greps for
 function definitions across the engine will miss them unless it accounts for
 that.
+
+## [`common.func`](../vm/common.func)
+
+Helpers a VM script needs whatever the hypervisor. Both
+[`pve/vm-core.func`](../pve/vm-core.func) and
+[`incus/vm-core.func`](../incus/vm-core.func) load it, so a script finds the
+same functions on either host. Nothing in it calls `qm` or `incus`.
+
+| Function | Contract |
+| --- | --- |
+| `vm_require_tools <cmd>...` | Installs the packages for missing commands (`jq`, `xz`, `virt-customize`, `unzip`, `zstd`, `7z`, …) before anything is created; returns non-zero, with the reason, when it cannot. |
+| `vm_customize <label> <image> <virt-customize args>...` | A script's own image step. Warns with libguestfs' reason like the shared preparation steps, and returns non-zero so a required step can stop the run. |
+| `vm_release_asset <github\|gitlab\|codeberg> <owner/repo> <asset regex> [tag]` | Resolves a release through [`lib/forge.func`](../lib/forge.func) — tokens, rate limits and the API-less GitHub fallback included. Sets `VM_RELEASE_TAG`, `VM_RELEASE_VERSION`, `VM_RELEASE_ASSET`, `VM_RELEASE_URL` and, from GitHub's asset digest, `VM_RELEASE_SHA256`. |
+| `vm_latest_from_index <url> <pcre> [--sort-by <pcre>] [--probe <url with {}>]...` | Newest match on an HTML download index (`grep -oP`, so `\K` works). `--sort-by` orders by part of the match; each `--probe` template is tried newest first and the first answering URL wins. Sets `VM_INDEX_LATEST` and `VM_INDEX_URL`. Never falls back to a hardcoded release. |
+| `vm_firstboot_unit <image> <name> <script> [--description …] [--after <unit>]... [--requires-path <path>] [--cloud-init yes\|no]` | Stages `<script>` as `/usr/local/sbin/<name>` plus a run-once `<name>.service`. The marker `/var/lib/community-scripts/<name>.done` (`VM_FIRSTBOOT_MARKER`) is written only after success, so a failure is retried on the next boot. With Cloud-Init the unit is wanted by `cloud-init.target` and ordered after `cloud-final.service`; wanting `multi-user.target` there would create an ordering cycle. |
+| `vm_wait_http <url> [timeout] [--insecure]` | Succeeds on any 2xx/3xx, or 401/403 from a login page. |
+| `vm_print_summary ["Label=value"]...`, `vm_next_steps <step>...`, `vm_finish [message]` | The standard closing block: VM ID, hostname, resources, state and `VM_IP`, extra rows, numbered next steps, then the `done` telemetry and the completion message. |
+
+Checksums: `vm_fetch_image` verifies the vendor checksum for Debian, Ubuntu,
+TrueNAS, OpenWrt (`sha256sums`), FreeBSD VM images (`CHECKSUM.SHA256`) and
+TurnKey (`.hash`) by itself. Elsewhere pass `--sha256` (for example
+`VM_RELEASE_SHA256`) or `--checksum-url <sums file>`; GNU, BSD and bare-hash
+layouts are understood and the algorithm follows from the hash length.
+
+### Proxmox-only counterparts
+
+[`pve/vm-core.func`](../pve/vm-core.func) adds the steps that drive `qm`:
+
+| Function | Contract |
+| --- | --- |
+| `vm_import_disk <vmid> <image> <storage> [format]` | `qm disk import` (or `qm importdisk` on older hosts) with `DISK_IMPORT_FORMAT`; sets `VM_IMPORTED_DISK` to the volume Proxmox reported or the `unusedN` entry the import added. Never guesses a name. |
+| `vm_start_vm [label]` | Starts the VM when `START_VM=yes`. |
+| `vm_wait_guest_agent <vmid> [timeout]` | Waits until the guest agent answers. |
+| `vm_guest_exec <vmid> <timeout> [--] <cmd>...` | Runs a command through the guest agent; prints its stdout and returns its exit code (124: still running). |
+| `vm_wait_for_ip [timeout]` | Sets `VM_IP` from the NIC matching `MAC`; warns and returns non-zero when there is none yet. |
+
+On Incus these fail with exit 119 and say why: `incus_vm_create` imports,
+starts and waits for the agent itself.
+
+Run the host-independent regression tests with `bash tests/vm-helpers.sh`.

incus/vm-core.func

@@ -45,6 +45,8 @@ _incus_vm_source "core/core.func"
 # Value checks that do not depend on the platform — the Proxmox VE VM path
 # loads the same file, so an MTU means the same thing on both.
 _incus_vm_source "host/validate.func"
+# Helpers that do not touch qm or incus, shared with pve/vm-core.func.
+_incus_vm_source "vm/common.func"
 _incus_vm_source "incus/core.func"
 load_incus_functions
 incus_install_compat_aliases
@@ -871,6 +873,18 @@ vm_select_iso_storage() {
   return 119
 }
 
+# The qm-driven steps have no Incus counterpart: incus_vm_create imports,
+# starts and waits for the agent itself.
+_incus_vm_qm_only() {
+  msg_error "$1 drives qm and is only available on Proxmox VE; use incus_vm_create on Incus."
+  return 119
+}
+vm_import_disk() { _incus_vm_qm_only vm_import_disk; }
+vm_start_vm() { _incus_vm_qm_only vm_start_vm; }
+vm_wait_guest_agent() { _incus_vm_qm_only vm_wait_guest_agent; }
+vm_guest_exec() { _incus_vm_qm_only vm_guest_exec; }
+vm_wait_for_ip() { _incus_vm_qm_only vm_wait_for_ip; }
+
 # The Proxmox counterpart picks a disk format from the storage type because qm
 # importdisk has to be told one. Incus decides the on-pool layout itself and
 # never exposes it, so the only thing left to decide is the format the image
@@ -1252,26 +1266,21 @@ check_hostname_conflict() {
 # which guest agent is worth installing.
 
 vm_image_checksum() {
-  local url="${1:?url}" tool sums style want
+  local url="${1:?url}" name sums
 
+  name="${url%%\?*}"
+  name="${name##*/}"
   case "$url" in
-  *cloud.debian.org/*) tool=sha512sum; sums="${url%/*}/SHA512SUMS"; style=sums ;;
-  *cloud-images.ubuntu.com/*) tool=sha256sum; sums="${url%/*}/SHA256SUMS"; style=sums ;;
-  *download.truenas.com/*) tool=sha256sum; sums="${url}.sha256"; style=bare ;;
+  *cloud.debian.org/*) sums="${url%/*}/SHA512SUMS" ;;
+  *cloud-images.ubuntu.com/*) sums="${url%/*}/SHA256SUMS" ;;
+  *download.truenas.com/*) sums="${url}.sha256" ;;
+  *downloads.openwrt.org/*) sums="${url%/*}/sha256sums" ;;
+  *download.freebsd.org/*/VM-IMAGES/*) sums="${url%/*}/CHECKSUM.SHA256" ;;
+  *mirror.turnkeylinux.org/*) sums="${url}.hash" ;;
   *) return 1 ;;
   esac
 
-  command -v "$tool" >/dev/null 2>&1 || return 1
-
-  if [[ "$style" == bare ]]; then
-    want=$(curl -fsSL --max-time 30 "$sums" 2>/dev/null | tr -d '[:space:]')
-  else
-    want=$(curl -fsSL --max-time 30 "$sums" 2>/dev/null |
-      awk -v f="$(basename "$url")" '$NF == f || $NF == "*" f {print $1; exit}')
-  fi
-
-  [[ "$want" =~ ^([0-9a-fA-F]{64}|[0-9a-fA-F]{128})$ ]] || return 1
-  printf '%s %s' "$tool" "$want"
+  _vm_checksum_from_sums "$sums" "$name"
 }
 
 _vm_image_ok() {
@@ -1312,12 +1321,13 @@ vm_image_cache_path() {
 
 # vm_fetch_image <url> <target> [--cache] [--verify-xz] [--min-bytes N]
 #                               [--exact-bytes N] [--sha256 H] [--sha1 H]
-#                               [--advisory-sha1 H] [--needs-bytes N]
+#                               [--checksum-url URL] [--advisory-sha1 H]
+#                               [--needs-bytes N]
 vm_fetch_image() {
   local url="${1:?url}" target="${2:?target}"
   shift 2
   local min_bytes=0 exact_bytes="" sha256="" sha1="" advisory_sha1="" needs_bytes=""
-  local cache=no xz=no
+  local checksum_url="" cache=no xz=no
   local got avail_mib need_mib have part sha_tool="" sha_want="" discovered
 
   while [[ $# -gt 0 ]]; do
@@ -1328,16 +1338,23 @@ vm_fetch_image() {
     --exact-bytes) exact_bytes="$2"; shift 2 ;;
     --sha256) sha256="$2"; shift 2 ;;
     --sha1) sha1="$2"; shift 2 ;;
+    --checksum-url) checksum_url="$2"; shift 2 ;;
     --advisory-sha1) advisory_sha1="$2"; shift 2 ;;
     --needs-bytes) needs_bytes="$2"; shift 2 ;;
     *) msg_error "vm_fetch_image: unknown option $1"; return 2 ;;
     esac
   done
 
   if [[ -n "$sha256" ]]; then
-    sha_tool=sha256sum sha_want="$sha256"
+    sha_tool=sha256sum sha_want="${sha256,,}"
   elif [[ -n "$sha1" ]]; then
     sha_tool=sha1sum sha_want="$sha1"
+  elif [[ -n "$checksum_url" ]]; then
+    if discovered=$(_vm_checksum_from_sums "$checksum_url" "$(basename "$target")"); then
+      read -r sha_tool sha_want <<<"$discovered"
+    else
+      msg_warn "No checksum for $(basename "$target") in ${checksum_url}; checking the size only"
+    fi
   elif discovered=$(vm_image_checksum "$url"); then
     read -r sha_tool sha_want <<<"$discovered"
   fi

pve/vm-app.func

@@ -294,18 +294,14 @@ get_image_url() {
 download_and_cache_image() {
   msg_info "Retrieving the URL for the ${OS_DISPLAY} Qcow2 Disk Image"
   URL=$(get_image_url)
-  CACHE_DIR="/var/lib/vz/template/cache"
-  CACHE_FILE="$CACHE_DIR/$(basename "$URL")"
-  mkdir -p "$CACHE_DIR"
+  CACHE_FILE="$(vm_image_cache_path "$URL")"
+  CACHE_DIR="$(dirname "$CACHE_FILE")"
   msg_ok "${CL}${BL}${URL}${CL}"
 
-  if [[ ! -s "$CACHE_FILE" ]]; then
-    curl -f#SL -o "$CACHE_FILE" "$URL"
-    echo -en "\e[1A\e[0K"
-    msg_ok "Downloaded ${CL}${BL}$(basename "$CACHE_FILE")${CL}"
-  else
-    msg_ok "Using cached image ${CL}${BL}$(basename "$CACHE_FILE")${CL}"
-  fi
+  # Through vm_fetch_image: a partial file is never mistaken for the image, and
+  # the vendor checksum is verified where one is published.
+  vm_fetch_image "$URL" "$CACHE_FILE" --cache --min-bytes $((10 * 1024 * 1024)) || exit 1
+  msg_ok "Image ready ${CL}${BL}$(basename "$CACHE_FILE")${CL}"
 }
 
 # ==============================================================================
@@ -318,12 +314,7 @@ download_and_cache_image() {
 # Ensures virt-customize (libguestfs-tools) is installed on the Proxmox host.
 # ------------------------------------------------------------------------------
 ensure_virt_customize() {
-  if ! command -v virt-customize &>/dev/null; then
-    msg_info "Installing libguestfs-tools"
-    apt-get -qq update >/dev/null
-    apt-get -qq install libguestfs-tools -y >/dev/null
-    msg_ok "Installed libguestfs-tools"
-  fi
+  vm_require_tools virt-customize || exit 1
 }
 
 # ------------------------------------------------------------------------------
@@ -467,22 +458,8 @@ create_app_vm() {
   msg_ok "Created VM shell"
 
   # Import disk
-  msg_info "Importing disk into storage ($STORAGE)"
-  if qm disk import --help >/dev/null 2>&1; then
-    IMPORT_CMD=(qm disk import)
-  else
-    IMPORT_CMD=(qm importdisk)
-  fi
-
-  IMPORT_OUT="$("${IMPORT_CMD[@]}" "$VMID" "$WORK_FILE" "$STORAGE" ${DISK_IMPORT:-} 2>&1 || true)"
-  DISK_REF_IMPORTED="$(printf '%s\n' "$IMPORT_OUT" | sed -n "s/.*successfully imported disk '\([^']\+\)'.*/\1/p" | tr -d "\r\"'")"
-  [[ -z "$DISK_REF_IMPORTED" ]] && DISK_REF_IMPORTED="$(pvesm list "$STORAGE" | awk -v id="$VMID" '$5 ~ ("vm-"id"-disk-") {print $1":"$5}' | sort | tail -n1)"
-  [[ -z "$DISK_REF_IMPORTED" ]] && {
-    msg_error "Unable to determine imported disk reference."
-    echo "$IMPORT_OUT"
-    exit 1
-  }
-  msg_ok "Imported disk (${CL}${BL}${DISK_REF_IMPORTED}${CL})"
+  vm_import_disk "$VMID" "$WORK_FILE" "$STORAGE" || exit 1
+  DISK_REF_IMPORTED="$VM_IMPORTED_DISK"
 
   # Clean up work file
   rm -f "$WORK_FILE"
@@ -537,15 +514,12 @@ wait_for_vm_ip() {
     return 0
   fi
 
-  set +e
-  for i in {1..20}; do
-    VM_IP=$(qm guest cmd "$VMID" network-get-interfaces 2>/dev/null |
-      jq -r '.[] | select(.name != "lo") | ."ip-addresses"[]? | select(."ip-address-type" == "ipv4") | ."ip-address"' 2>/dev/null |
-      grep -v "^127\." | head -1) || true
-    [ -n "$VM_IP" ] && break
-    sleep 3
-  done
-  set -e
+  # Matched by the NIC's MAC: the app may run Docker, whose bridge address
+  # used to be reported as the VM's.
+  if load_cloud_init_functions; then
+    VM_IP="$(get_vm_ip "$VMID" 60 "${MAC:-}")" || VM_IP=""
+  fi
+  return 0
 }
 
 # ------------------------------------------------------------------------------