From a908fd3a8e24e1f0f1321d45f3a7d0ed106cccd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Thu, 25 Jun 2026 11:18:00 +0200 Subject: [PATCH 01/10] storage: enable RBD/Ceph volume encryption support (shared base) Flip StoragePoolType.RBD from EncryptionSupport.Unsupported to Hypervisor so the existing encryption control plane (allocator, endpoint selector, offerings) treats RBD pools as encryption-capable. The agent-side encrypted RBD create path is not implemented yet; it is delivered by two follow-up tracks (qemu-native engine='qemu' and ceph-native engine='librbd'). Until then, fail closed at the two RBD create chokepoints in LibvirtStorageAdaptor (createPhysicalDisk and createDiskFromTemplate) when a passphrase is present, so we never silently produce a plaintext volume that the control plane believes is encrypted. No change for existing unencrypted RBD volumes (guards only fire when a passphrase is set; supportsEncryption() only affects volumes that require encryption). --- api/src/main/java/com/cloud/storage/Storage.java | 2 +- .../kvm/storage/LibvirtStorageAdaptor.java | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index ddf5978497ba..0f7f12700f27 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -171,7 +171,7 @@ public static enum StoragePoolType { LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR CLVM(true, false, EncryptionSupport.Unsupported), CLVM_NG(true, false, EncryptionSupport.Hypervisor), - RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD + RBD(true, true, EncryptionSupport.Hypervisor), // http://libvirt.org/storage.html#StorageBackendRBD ; encrypted natively by librbd (LUKS2, engine='librbd') SharedMountPoint(true, true, EncryptionSupport.Hypervisor), VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers. diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index db37a7e948c4..015225fb9e53 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -985,6 +985,13 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, StoragePoolType poolType = pool.getType(); if (StoragePoolType.RBD.equals(poolType)) { + // RBD is advertised as encryption-capable (StoragePoolType.RBD -> EncryptionSupport.Hypervisor), + // but the encrypted RBD create path is not implemented on this branch yet. Fail closed so we never + // silently create a plaintext volume that the control plane believes is encrypted. + // The qemu-native (engine='qemu') and ceph-native (engine='librbd') tracks each replace this guard. + if (passphrase != null && passphrase.length > 0) { + throw new CloudRuntimeException("Encrypted volume creation on RBD is not yet implemented (volume " + name + ")"); + } Map details = pool.getDetails(); String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); @@ -1244,6 +1251,12 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, KVMPhysicalDisk disk = null; if (destPool.getType() == StoragePoolType.RBD) { + // See createPhysicalDisk(): encrypted root-disk-from-template on RBD is not implemented yet. + // createDiskFromTemplateOnRBD() does not consume the passphrase, so fail closed rather than + // silently producing a plaintext root volume. Tracks A/B replace this with real create logic. + if (passphrase != null && passphrase.length > 0) { + throw new CloudRuntimeException("Encrypted root volume creation from template on RBD is not yet implemented (volume " + name + ")"); + } disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout); } else { try (KeyFile keyFile = new KeyFile(passphrase)){ From 8c692ca6ff4d1f10c3dc0addc65a4e2263319638 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Thu, 25 Jun 2026 12:25:10 +0200 Subject: [PATCH 02/10] kvm: Ceph-native LUKS2 encryption for RBD volumes (engine='librbd') Implements encrypted RBD data and root disks using librbd's native LUKS2 encryption, decrypted at runtime by libvirt/qemu via . CloudStack manages the passphrase (existing model). - RbdEncryption: isolated helper wrapping `rbd encryption format luks2`, cephx via --id + keyfile (secret not on the command line), LUKS passphrase via KeyFile. Kept separate so the CLI can later be swapped for a JNA binding (rados-java has no rbd_encryption_format API). - LibvirtStorageAdaptor: create/clone the raw RBD image, then apply `rbd encryption format luks2`; mark the disk LUKS2 so encrypt_format propagates to the volume. Replaces the fail-closed guards. - QemuObject.EncryptFormat: add LUKS2. - LibvirtVMDef: render ; the encrypt details now carry an optional engine. - attach (KVMStorageProcessor) and boot (LibvirtComputingResource): set engine='librbd' for RBD-backed encrypted volumes. NOTE: the CoW-clone-then-format path (encrypted root from an unencrypted template) needs live-cluster validation for the parent-grow / usable-size behaviour described in the Ceph image-encryption docs. Builds: api + plugins/hypervisors/kvm (JDK11). --- .../resource/LibvirtComputingResource.java | 4 +- .../hypervisor/kvm/resource/LibvirtVMDef.java | 13 ++- .../kvm/storage/KVMStorageProcessor.java | 4 +- .../kvm/storage/LibvirtStorageAdaptor.java | 51 ++++++--- .../cloudstack/utils/qemu/QemuObject.java | 1 + .../cloudstack/utils/rbd/RbdEncryption.java | 107 ++++++++++++++++++ 6 files changed, 160 insertions(+), 20 deletions(-) create mode 100644 plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 41716881fa4a..e86a716ba731 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3864,7 +3864,9 @@ public int compare(final DiskTO arg0, final DiskTO arg1) { if (volumeObjectTO.requiresEncryption() && pool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor ) { String secretUuid = createLibvirtVolumeSecret(conn, volumeObjectTO.getPath(), volumeObjectTO.getPassphrase()); - DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat())); + // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine. + String encryptEngine = (pool.getType() == StoragePoolType.RBD) ? "librbd" : null; + DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat()), encryptEngine); disk.setLibvirtDiskEncryptDetails(encryptDetails); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 7f6725b6d152..2f7702a5db8b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -788,14 +788,21 @@ public static class DiskDef { public static class LibvirtDiskEncryptDetails { String passphraseUuid; QemuObject.EncryptFormat encryptFormat; + String engine; // optional libvirt encryption engine (e.g. "librbd"); null => libvirt/qemu default public LibvirtDiskEncryptDetails(String passphraseUuid, QemuObject.EncryptFormat encryptFormat) { + this(passphraseUuid, encryptFormat, null); + } + + public LibvirtDiskEncryptDetails(String passphraseUuid, QemuObject.EncryptFormat encryptFormat, String engine) { this.passphraseUuid = passphraseUuid; this.encryptFormat = encryptFormat; + this.engine = engine; } public String getPassphraseUuid() { return this.passphraseUuid; } public QemuObject.EncryptFormat getEncryptFormat() { return this.encryptFormat; } + public String getEngine() { return this.engine; } } public static class DiskGeometry { @@ -1437,7 +1444,11 @@ public String toString() { } if (encryptDetails != null) { - diskBuilder.append("\n"); + diskBuilder.append("\n"); diskBuilder.append("\n"); diskBuilder.append("\n"); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 009e1decee2b..ba820a3a6c58 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -1802,7 +1802,9 @@ public Answer attachVolume(final AttachCommand cmd) { DiskDef.LibvirtDiskEncryptDetails encryptDetails = null; if (vol.requiresEncryption()) { String secretUuid = resource.createLibvirtVolumeSecret(conn, vol.getPath(), vol.getPassphrase()); - encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(vol.getEncryptFormat())); + // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine. + String encryptEngine = (primaryStore.getPoolType() == StoragePoolType.RBD) ? "librbd" : null; + encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(vol.getEncryptFormat()), encryptEngine); vol.clearPassphrase(); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 015225fb9e53..498808f04b1d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -34,7 +34,9 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.utils.cryptsetup.CryptSetup; import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.apache.cloudstack.utils.qemu.QemuImageOptions; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; @@ -985,18 +987,21 @@ public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, StoragePoolType poolType = pool.getType(); if (StoragePoolType.RBD.equals(poolType)) { - // RBD is advertised as encryption-capable (StoragePoolType.RBD -> EncryptionSupport.Hypervisor), - // but the encrypted RBD create path is not implemented on this branch yet. Fail closed so we never - // silently create a plaintext volume that the control plane believes is encrypted. - // The qemu-native (engine='qemu') and ceph-native (engine='librbd') tracks each replace this guard. - if (passphrase != null && passphrase.length > 0) { - throw new CloudRuntimeException("Encrypted volume creation on RBD is not yet implemented (volume " + name + ")"); - } Map details = pool.getDetails(); String dataPool = (details == null) ? null : details.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); - return (dataPool == null) ? createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : - createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, passphrase); + // Create the raw RBD image first. For encrypted volumes we apply a native librbd LUKS header + // afterwards via `rbd encryption format` (engine='librbd'). We deliberately do NOT hand the + // passphrase to qemu-img, which would instead produce a qemu-native LUKS container. + KVMPhysicalDisk disk = (dataPool == null) ? + createPhysicalDiskByLibVirt(name, pool, PhysicalDiskFormat.RAW, provisioningType, size) : + createPhysicalDiskByQemuImg(name, pool, PhysicalDiskFormat.RAW, provisioningType, size, null); + + if (passphrase != null && passphrase.length > 0) { + formatRbdImageEncryption(pool, name, passphrase); + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + } + return disk; } else if (QEMU_IMG_MANAGED_POOL_TYPES.contains(poolType)) { switch (format) { case QCOW2: @@ -1251,13 +1256,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, KVMPhysicalDisk disk = null; if (destPool.getType() == StoragePoolType.RBD) { - // See createPhysicalDisk(): encrypted root-disk-from-template on RBD is not implemented yet. - // createDiskFromTemplateOnRBD() does not consume the passphrase, so fail closed rather than - // silently producing a plaintext root volume. Tracks A/B replace this with real create logic. - if (passphrase != null && passphrase.length > 0) { - throw new CloudRuntimeException("Encrypted root volume creation from template on RBD is not yet implemented (volume " + name + ")"); - } - disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout); + disk = createDiskFromTemplateOnRBD(template, name, format, provisioningType, size, destPool, timeout, passphrase); } else { try (KeyFile keyFile = new KeyFile(passphrase)){ String newUuid = name; @@ -1337,7 +1336,7 @@ public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, } private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, - String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout){ + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, KVMStoragePool destPool, int timeout, byte[] passphrase){ /* With RBD you can't run qemu-img convert with an existing RBD image as destination @@ -1506,9 +1505,27 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, disk = null; } } + + if (disk != null && passphrase != null && passphrase.length > 0) { + // Apply native librbd LUKS to the freshly created/cloned root image (engine='librbd'). + // NOTE: when the destination is a CoW clone of an (unencrypted) template, `rbd encryption + // format` reserves LUKS-header space; the parent-grow / usable-size behaviour described in the + // Ceph "Image Encryption" docs must be validated against a live cluster (see spec B gotchas). + formatRbdImageEncryption(destPool, newUuid, passphrase); + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + } return disk; } + /** + * Apply native librbd LUKS encryption to an existing RBD image via the rbd CLI. + * Isolated here so the CLI dependency can later be swapped for a native (JNA) librbd binding. + */ + private void formatRbdImageEncryption(KVMStoragePool pool, String image, byte[] passphrase) { + new RbdEncryption().format(pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getAuthSecret(), pool.getSourceDir(), image, passphrase, CryptSetup.LuksType.LUKS2); + } + @Override public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool) { return null; diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java index efeee04cb90f..511e91074969 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuObject.java @@ -54,6 +54,7 @@ public enum ObjectParameter { */ public enum EncryptFormat { LUKS("luks"), + LUKS2("luks2"), AES("aes"); private final String format; diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java new file mode 100644 index 000000000000..a062334cce25 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.utils.rbd; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.cryptsetup.CryptSetup; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** + * Thin wrapper around the {@code rbd} CLI to apply native librbd LUKS encryption to an + * RBD image via {@code rbd encryption format}. This is only used at volume create time; + * runtime decryption is handled by libvirt/qemu through {@code }. + * + * The CLI dependency is intentionally isolated in this class so it can later be replaced + * by a native librbd (JNA) binding without touching callers. rados-java (0.x) does not + * expose the rbd_encryption_format API, hence the CLI for now. + */ +public class RbdEncryption { + protected static Logger LOGGER = LogManager.getLogger(RbdEncryption.class); + + protected String commandPath = "rbd"; + + public RbdEncryption() {} + + public RbdEncryption(String commandPath) { + this.commandPath = commandPath; + } + + /** + * Apply a LUKS header to an existing RBD image so librbd can transparently encrypt it. + *

+ * cephx authentication is supplied via {@code --id} plus a temporary keyfile so the secret + * never appears on the command line. The LUKS passphrase is supplied via a temporary file + * ({@link KeyFile}). Both temp files are deleted when this method returns. + * + * @param monHost ceph monitor host + * @param monPort ceph monitor port (0 to omit) + * @param authUser cephx user (e.g. "cloudstack"); null to skip --id + * @param authSecret cephx secret/key (as used for ceph "key" config); null to skip --keyfile + * @param cephPool ceph pool name + * @param image rbd image name + * @param passphrase LUKS passphrase + * @param luksType LUKS1/LUKS2 (librbd engine supports both; LUKS2 recommended) + */ + public void format(String monHost, int monPort, String authUser, String authSecret, + String cephPool, String image, byte[] passphrase, CryptSetup.LuksType luksType) { + final String imageSpec = cephPool + "/" + image; + try (KeyFile passFile = new KeyFile(passphrase); + KeyFile cephKeyFile = new KeyFile(authSecret == null ? null : authSecret.getBytes(StandardCharsets.UTF_8))) { + final Script script = new Script(commandPath); + script.add("encryption"); + script.add("format"); + script.add(imageSpec); + script.add(luksType.toString()); + script.add(passFile.toString()); + script.add("--mon-host"); + script.add(monPort > 0 ? monHost + ":" + monPort : monHost); + if (authUser != null) { + script.add("--id"); + script.add(authUser); + } + if (cephKeyFile.isSet()) { + script.add("--keyfile"); + script.add(cephKeyFile.toString()); + } + + final String result = script.execute(); + if (result != null) { + throw new CloudRuntimeException(String.format("Failed to apply librbd %s encryption to %s: %s", luksType, imageSpec, result)); + } + LOGGER.debug("Applied {} encryption to RBD image {}", luksType, imageSpec); + } catch (IOException ex) { + throw new CloudRuntimeException(String.format("Failed to apply librbd %s encryption to %s", luksType, imageSpec), ex); + } + } + + /** + * Best-effort probe that the local rbd CLI supports the encryption subcommand. + */ + public boolean isSupported() { + final Script script = new Script(commandPath); + script.add("help"); + script.add("encryption"); + script.add("format"); + return script.execute() == null; + } +} From e547f86a7afe640936c41e36e703686575c1badb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Thu, 25 Jun 2026 13:01:34 +0200 Subject: [PATCH 03/10] kvm: gate host encryption probe on librbd support for RBD hostSupportsVolumeEncryption() now advertises encryption capability if the host supports EITHER qemu-native LUKS (qemu-img LUKS + cryptsetup) OR librbd native encryption (rbd CLI with the encryption subcommand). Previously a Ceph-only host that lacked cryptsetup would not advertise encryption even though librbd can encrypt RBD volumes. Split into hostSupportsQemuNativeVolumeEncryption() and hostSupportsRbdVolumeEncryption(); kept HOST_VOLUME_ENCRYPTION as the single host-wide flag (documented limitation: not per-pool). --- .../resource/LibvirtComputingResource.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index e86a716ba731..fba3f22905e7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -89,6 +89,7 @@ import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.cryptsetup.CryptSetup; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; import org.apache.cloudstack.utils.linux.CPUStat; import org.apache.cloudstack.utils.linux.KVMHostInfo; @@ -6166,10 +6167,28 @@ public boolean isHostSecured() { } /** - * Test host for volume encryption support + * Test host for volume encryption support. A host is considered encryption-capable if it + * supports EITHER mechanism CloudStack can use: + * - qemu-native LUKS (qemu-img LUKS + cryptsetup) for file/block backed pools, or + * - librbd native encryption (rbd encryption format) for RBD/Ceph pools. + * NOTE: HOST_VOLUME_ENCRYPTION is a single host-wide flag and is not per-pool, so a host that + * advertises encryption via only one mechanism could still be selected for a volume that needs + * the other. In practice hosts that do encryption have the qemu-native stack; the librbd branch + * additionally covers Ceph-only hosts. * @return boolean */ public boolean hostSupportsVolumeEncryption() { + boolean supported = hostSupportsQemuNativeVolumeEncryption() || hostSupportsRbdVolumeEncryption(); + if (!supported) { + LOGGER.info("Host does not support volume encryption (no qemu-native LUKS + cryptsetup, and no librbd rbd encryption)"); + } + return supported; + } + + /** + * Test host for qemu-native LUKS volume encryption (qemu-img LUKS support + cryptsetup). + */ + public boolean hostSupportsQemuNativeVolumeEncryption() { // test qemu-img try { QemuImg qemu = new QemuImg(0); @@ -6191,6 +6210,13 @@ public boolean hostSupportsVolumeEncryption() { return true; } + /** + * Test host for librbd native LUKS encryption support (rbd CLI with the encryption subcommand). + */ + public boolean hostSupportsRbdVolumeEncryption() { + return new RbdEncryption().isSupported(); + } + public boolean isSecureMode(String bootMode) { if (StringUtils.isNotBlank(bootMode) && "secure".equalsIgnoreCase(bootMode)) { return true; From 864731cb004ad2e46c6751f1fad93898b40f01ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Thu, 25 Jun 2026 13:12:27 +0200 Subject: [PATCH 04/10] kvm: resize support for librbd-encrypted RBD volumes (#5) Encrypted RBD volumes are encrypted natively by librbd and must be resized with `rbd resize --encryption-passphrase-file` so librbd grows the encrypted payload and keeps the LUKS header consistent. The existing encrypted-resize path (resizeEncryptedQcowFile) uses qemu-img --object secret, which is for qemu-native LUKS and does not fit the librbd LUKS2 layout. - RbdEncryption.resize(): new `rbd resize` wrapper (cephx via --id + keyfile, passphrase via KeyFile, optional --allow-shrink). - LibvirtResizeVolumeCommandWrapper: detect encrypted RBD and route to the rbd resize path, bypassing the libvirt v.resize and qemu-img paths. Snapshot/revert, RBD<->RBD copy, and migration of encrypted RBD volumes need no code changes: they operate on the raw (LUKS-containing) image at the block level, and the destination passphrase secret is already created engine-agnostic in LibvirtPrepareForMigrationCommandWrapper. These still require live validation. Builds: plugins/hypervisors/kvm (JDK11). --- .../LibvirtResizeVolumeCommandWrapper.java | 27 +++++++++++- .../cloudstack/utils/rbd/RbdEncryption.java | 44 +++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index a43b584dd6d6..2a1f9cb0cd9e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.cloudstack.utils.rbd.RbdEncryption; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo; @@ -93,6 +94,13 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR final String path = vol.getPath(); String type = notifyOnlyType; + // Encrypted RBD volumes are encrypted natively by librbd; they must be resized via + // `rbd resize --encryption-passphrase-file` so librbd grows the encrypted payload and keeps + // the LUKS header consistent. The libvirt/qemu-img resize paths below are for qemu-native + // encryption and would not handle the librbd LUKS2 layout. + final boolean rbdEncrypted = pool.getType() == StoragePoolType.RBD + && command.getPassphrase() != null && command.getPassphrase().length > 0; + if (spool.getType().equals(StoragePoolType.PowerFlex) && vol.getFormat().equals(PhysicalDiskFormat.QCOW2)) { // PowerFlex QCOW2 sizing needs to consider overhead. newSize = ScaleIOStorageAdaptor.getUsableBytesFromRawBytes(newSize); @@ -115,7 +123,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR /* libvirt doesn't support resizing (C)LVM devices, and corrupts QCOW2 in some scenarios, so we have to do these via qemu-img */ if (pool.getType() != StoragePoolType.CLVM && pool.getType() != StoragePoolType.CLVM_NG && pool.getType() != StoragePoolType.Linstor && pool.getType() != StoragePoolType.PowerFlex - && vol.getFormat() != PhysicalDiskFormat.QCOW2) { + && vol.getFormat() != PhysicalDiskFormat.QCOW2 && !rbdEncrypted) { logger.debug("Volume " + path + " can be resized by libvirt. Asking libvirt to resize the volume."); try { final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); @@ -143,7 +151,12 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR If VM is online, the existing resize script will call virsh blockresize which works with both encrypted and non-encrypted volumes. */ - if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) { + if (rbdEncrypted) { + logger.debug("Invoking rbd to resize an encrypted (librbd) RBD volume"); + // NOTE: for a running VM this grows the rbd image, but the guest may not see the new size + // until the domain is notified/restarted. Online notification needs live validation. + resizeRbdEncryptedVolume(pool, vol, newSize, shrinkOk, command.getPassphrase()); + } else if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) { logger.debug("Invoking qemu-img to resize an offline, encrypted volume"); QemuObject.EncryptFormat encryptFormat = QemuObject.EncryptFormat.enumValue(command.getEncryptFormat()); resizeEncryptedQcowFile(vol, encryptFormat,newSize, command.getPassphrase(), libvirtComputingResource); @@ -213,6 +226,16 @@ private void resizeEncryptedQcowFile(final KVMPhysicalDisk vol, final QemuObject } } + private void resizeRbdEncryptedVolume(final KVMStoragePool pool, final KVMPhysicalDisk vol, long newSize, + boolean shrinkOk, byte[] passphrase) throws CloudRuntimeException { + try { + new RbdEncryption().resize(pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), + pool.getAuthSecret(), pool.getSourceDir(), vol.getName(), newSize, shrinkOk, passphrase); + } finally { + Arrays.fill(passphrase, (byte) 0); + } + } + private Answer handleMultipathSCSIResize(ResizeVolumeCommand command, KVMStoragePool pool) { ((MultipathSCSIPool)pool).resize(command.getPath(), command.getInstanceName(), command.getNewSize()); return new ResizeVolumeAnswer(command, true, ""); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java index a062334cce25..c4f7660cf008 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java @@ -94,6 +94,50 @@ public void format(String monHost, int monPort, String authUser, String authSecr } } + /** + * Resize an encrypted RBD image. librbd needs the passphrase so it can resize the encrypted + * payload (not just the raw image) and keep the LUKS header consistent. {@code newSizeBytes} is + * the usable (decrypted) size requested; rbd {@code --size} is expressed in MiB. + * + * @param allowShrink pass --allow-shrink when shrinking is permitted + */ + public void resize(String monHost, int monPort, String authUser, String authSecret, + String cephPool, String image, long newSizeBytes, boolean allowShrink, byte[] passphrase) { + final String imageSpec = cephPool + "/" + image; + final long sizeMiB = newSizeBytes / (1024L * 1024L); + try (KeyFile passFile = new KeyFile(passphrase); + KeyFile cephKeyFile = new KeyFile(authSecret == null ? null : authSecret.getBytes(StandardCharsets.UTF_8))) { + final Script script = new Script(commandPath); + script.add("resize"); + script.add("--size"); + script.add(String.valueOf(sizeMiB)); + script.add(imageSpec); + script.add("--encryption-passphrase-file"); + script.add(passFile.toString()); + if (allowShrink) { + script.add("--allow-shrink"); + } + script.add("--mon-host"); + script.add(monPort > 0 ? monHost + ":" + monPort : monHost); + if (authUser != null) { + script.add("--id"); + script.add(authUser); + } + if (cephKeyFile.isSet()) { + script.add("--keyfile"); + script.add(cephKeyFile.toString()); + } + + final String result = script.execute(); + if (result != null) { + throw new CloudRuntimeException(String.format("Failed to resize encrypted RBD image %s to %d MiB: %s", imageSpec, sizeMiB, result)); + } + LOGGER.debug("Resized encrypted RBD image {} to {} MiB", imageSpec, sizeMiB); + } catch (IOException ex) { + throw new CloudRuntimeException(String.format("Failed to resize encrypted RBD image %s", imageSpec), ex); + } + } + /** * Best-effort probe that the local rbd CLI supports the encryption subcommand. */ From a8de6a7a2bc8dd1594076d2a8c738197b5399149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Thu, 25 Jun 2026 13:24:57 +0200 Subject: [PATCH 05/10] kvm: route online resize of encrypted RBD through virsh blockresize For a running VM, an librbd-encrypted RBD volume must be resized in-band by qemu/librbd, not out-of-band by the rbd CLI. Gate the CLI rbd-resize path on !vmIsRunning so: - offline -> `rbd resize --encryption-passphrase-file` (librbd-aware), and - online -> existing NOTIFYONLY path -> virsh blockresize, where qemu's block_resize delegates to librbd to grow the encrypted payload and notify the guest in one step (no passphrase needed; qemu holds the secret). This avoids notify-less out-of-band growth and qemu/librbd size divergence while the image is open. Online behaviour still needs live validation that blockresize resizes the encrypted payload for engine='librbd' disks. --- .../wrapper/LibvirtResizeVolumeCommandWrapper.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index 2a1f9cb0cd9e..20e3891478b4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -147,14 +147,13 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR boolean vmIsRunning = isVmRunning(vmInstanceName, libvirtComputingResource); - /* when VM is offline, we use qemu-img directly to resize encrypted volumes. - If VM is online, the existing resize script will call virsh blockresize which works - with both encrypted and non-encrypted volumes. + /* when VM is offline, we use qemu-img (or rbd, for librbd-encrypted RBD) directly to resize + encrypted volumes. If VM is online, the existing resize script calls virsh blockresize, + which for an librbd-encrypted RBD disk lets qemu/librbd grow the encrypted payload and + notify the guest in one step (no passphrase needed, qemu already has the secret loaded). */ - if (rbdEncrypted) { - logger.debug("Invoking rbd to resize an encrypted (librbd) RBD volume"); - // NOTE: for a running VM this grows the rbd image, but the guest may not see the new size - // until the domain is notified/restarted. Online notification needs live validation. + if (rbdEncrypted && !vmIsRunning) { + logger.debug("Invoking rbd to resize an offline, encrypted (librbd) RBD volume"); resizeRbdEncryptedVolume(pool, vol, newSize, shrinkOk, command.getPassphrase()); } else if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) { logger.debug("Invoking qemu-img to resize an offline, encrypted volume"); From 3b68ab7c9874273cf250f5340a9564bca680f25a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Sat, 4 Jul 2026 09:54:00 +0200 Subject: [PATCH 06/10] kvm: encrypted RBD root disks (thin CoW clone + full-copy fallback) Root disks could not be encrypted: cloning a plaintext template and then `rbd encryption format`ing the clone leaves the inherited OS data unreadable (the LUKS header offsets it), so the guest could not mount root. Fix, in createDiskFromTemplateOnRBD, with two paths: - Option A (same-cluster cached RBD template): grow the template base to reserve LUKS2 header space, snapshot+protect it (cloudstack-base-snap-luks), clone from it, apply the LUKS2 header, resize the clone to the requested size. Inherited template data stays readable through the clone's encryption and the clone is a thin CoW image (only the header is written). - Option B (first-use / non-RBD template): create an empty image, apply a LUKS2 header, then import the template THROUGH the encryption layer via RbdEncryption.importTemplate (qemu-img convert -n into encrypt.key-secret). Correct but a full copy. Validated end-to-end on Ubuntu 26.04 / libvirt 12.0.0: both boot; A is thin (3.5 GiB provisioned, ~120 MiB used); LUKS2 verified at rest on Ceph. --- .../kvm/storage/LibvirtStorageAdaptor.java | 98 +++++++++++++++++-- .../cloudstack/utils/rbd/RbdEncryption.java | 71 ++++++++++++++ 2 files changed, 161 insertions(+), 8 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 498808f04b1d..e2761bbffa2c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -97,6 +97,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private static final int RBD_FEATURE_DEEP_FLATTEN = 32; public static final int RBD_FEATURES = RBD_FEATURE_LAYERING + RBD_FEATURE_EXCLUSIVE_LOCK + RBD_FEATURE_OBJECT_MAP + RBD_FEATURE_FAST_DIFF + RBD_FEATURE_DEEP_FLATTEN; private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */ + /* Space reserved at the front of an encrypted RBD image for the LUKS2 header/keyslots so the + usable (decrypted) size still matches the requested volume size. */ + private static final long LUKS2_HEADER_RESERVE_BYTES = 16L << 20; // 16 MiB private static final Set QEMU_IMG_MANAGED_POOL_TYPES = Set.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem, StoragePoolType.SharedMountPoint); @@ -1363,6 +1366,91 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } + if (passphrase != null && passphrase.length > 0) { + boolean sameClusterRbd = srcPool.getType() == StoragePoolType.RBD + && srcPool.getSourceHost().equals(destPool.getSourceHost()) + && srcPool.getSourceDir().equals(destPool.getSourceDir()); + + if (sameClusterRbd) { + /* + * Option A (thin CoW encrypted root). Per the Ceph "Image Encryption" clone recipe: + * grow the template base to reserve LUKS2-header space, snapshot+protect that grown state, + * clone from it, apply a LUKS2 header, then resize the clone to the requested size. The + * inherited (plaintext) template data stays readable through the clone's encryption, and + * the clone is a thin CoW image (only the header is written, not the OS data). + */ + String encSnap = rbdTemplateSnapName + "-luks"; + try { + Rados r = new Rados(destPool.getAuthUserName()); + r.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); + r.confSet("key", destPool.getAuthSecret()); + r.confSet("client_mount_timeout", "30"); + r.connect(); + IoCTX io = r.ioCtxCreate(destPool.getSourceDir()); + Rbd rbd = new Rbd(io); + RbdImage base = rbd.open(template.getName()); + boolean haveEncSnap = false; + for (RbdSnapInfo s : base.snapList()) { + if (encSnap.equals(s.name)) { + haveEncSnap = true; + break; + } + } + if (!haveEncSnap) { + base.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); + base.snapCreate(encSnap); + base.snapProtect(encSnap); + logger.debug("Prepared LUKS-reserved template snapshot " + template.getName() + "@" + encSnap); + } + rbd.close(base); + rbd.clone(template.getName(), encSnap, io, newUuid, RBD_FEATURES, rbdOrder); + r.ioCtxDestroy(io); + } catch (RadosException | RbdException e) { + logger.error("Failed to create encrypted CoW clone " + newUuid + ": " + e.getMessage()); + return null; + } + formatRbdImageEncryption(destPool, newUuid, passphrase); + if (disk.getVirtualSize() > template.getVirtualSize()) { + // grow the clone to the requested root size (encryption-aware) + new RbdEncryption().resize(destPool.getSourceHost(), destPool.getSourcePort(), + destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), + newUuid, disk.getVirtualSize(), false, passphrase); + } + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + return disk; + } + + /* + * Option B (full-copy encrypted root) for templates not on the same RBD cluster (e.g. first + * use from secondary storage). Create an empty image, apply a LUKS2 header, then import the + * template THROUGH the encryption layer (qemu-img convert -n). Correct but not thin (no CoW). + */ + long createSize = disk.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES; + try { + Rados r = new Rados(destPool.getAuthUserName()); + r.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); + r.confSet("key", destPool.getAuthSecret()); + r.confSet("client_mount_timeout", "30"); + r.connect(); + IoCTX io = r.ioCtxCreate(destPool.getSourceDir()); + Rbd rbd = new Rbd(io); + rbd.create(newUuid, createSize, RBD_FEATURES, rbdOrder); + r.ioCtxDestroy(io); + } catch (RadosException | RbdException e) { + logger.error("Failed to create encrypted RBD image " + newUuid + ": " + e.getMessage()); + return null; + } + formatRbdImageEncryption(destPool, newUuid, passphrase); + boolean srcIsRbd = srcPool.getType() == StoragePoolType.RBD; + new RbdEncryption().importTemplate( + srcIsRbd ? srcPool.getSourceDir() : null, srcIsRbd ? template.getName() : null, + srcIsRbd ? null : template.getPath(), srcIsRbd ? null : template.getFormat().toString(), + destPool.getSourceHost(), destPool.getSourcePort(), destPool.getAuthUserName(), destPool.getAuthSecret(), + destPool.getSourceDir(), newUuid, passphrase, CryptSetup.LuksType.LUKS2); + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + return disk; + } + QemuImgFile srcFile; QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, disk.getPath())); destFile.setFormat(format); @@ -1506,14 +1594,8 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, } } - if (disk != null && passphrase != null && passphrase.length > 0) { - // Apply native librbd LUKS to the freshly created/cloned root image (engine='librbd'). - // NOTE: when the destination is a CoW clone of an (unencrypted) template, `rbd encryption - // format` reserves LUKS-header space; the parent-grow / usable-size behaviour described in the - // Ceph "Image Encryption" docs must be validated against a live cluster (see spec B gotchas). - formatRbdImageEncryption(destPool, newUuid, passphrase); - disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); - } + // Encrypted volumes are handled by the early return above (create empty -> luks2 format -> + // import template through encryption); the clone/convert path here is for plaintext volumes. return disk; } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java index c4f7660cf008..ce574ca70449 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java @@ -25,6 +25,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; /** * Thin wrapper around the {@code rbd} CLI to apply native librbd LUKS encryption to an @@ -39,6 +41,7 @@ public class RbdEncryption { protected static Logger LOGGER = LogManager.getLogger(RbdEncryption.class); protected String commandPath = "rbd"; + protected String qemuImgPath = "qemu-img"; public RbdEncryption() {} @@ -138,6 +141,74 @@ public void resize(String monHost, int monPort, String authUser, String authSecr } } + /** + * Import a template into an already-created, already-LUKS-formatted RBD image by writing it + * THROUGH the librbd encryption layer with {@code qemu-img convert -n} (so the data lands + * encrypted). This is how encrypted root disks are populated: we never clone-then-format a + * plaintext template (that leaves the inherited OS data unreadable) — instead we format an + * empty image and convert the template into it. + * + * Exactly one source must be given: an RBD image ({@code srcRbdPool}+{@code srcRbdImage}) or a + * local file ({@code srcFilePath}[+{@code srcFileFormat}]). cephx auth is provided to qemu-img + * via a temporary ceph.conf + keyring (deleted on return). + */ + public void importTemplate(String srcRbdPool, String srcRbdImage, + String srcFilePath, String srcFileFormat, + String monHost, int monPort, String authUser, String authSecret, + String cephPool, String destImage, byte[] passphrase, CryptSetup.LuksType luksType) { + final String imageSpec = cephPool + "/" + destImage; + final String monSpec = monPort > 0 ? monHost + ":" + monPort : monHost; + Path conf = null; + Path keyring = null; + try (KeyFile passFile = new KeyFile(passphrase)) { + keyring = Files.createTempFile("cs-ceph-", ".keyring"); // 0600 by default on POSIX + Files.writeString(keyring, "[client." + authUser + "]\n\tkey = " + authSecret + "\n"); + conf = Files.createTempFile("cs-ceph-", ".conf"); + Files.writeString(conf, "[global]\nmon_host = " + monSpec + "\nkeyring = " + keyring + "\n"); + + final Script q = new Script(qemuImgPath); + q.add("convert"); + q.add("-n"); // target already exists (pre-created + luks-formatted) + if (srcRbdImage != null) { + q.add("--image-opts"); + q.add("driver=rbd,pool=" + srcRbdPool + ",image=" + srcRbdImage + ",conf=" + conf + ",user=" + authUser); + } else { + if (srcFileFormat != null) { + q.add("-f"); + q.add(srcFileFormat.toLowerCase()); + } + q.add(srcFilePath); + } + q.add("--object"); + q.add("secret,id=luks0,file=" + passFile.toString()); + q.add("--target-image-opts"); + q.add("driver=rbd,pool=" + cephPool + ",image=" + destImage + ",conf=" + conf + ",user=" + authUser + + ",encrypt.format=" + luksType + ",encrypt.key-secret=luks0"); + + final String result = q.execute(); + if (result != null) { + throw new CloudRuntimeException(String.format("Failed to import template into encrypted RBD image %s: %s", imageSpec, result)); + } + LOGGER.debug("Imported template into encrypted RBD image {}", imageSpec); + } catch (IOException ex) { + throw new CloudRuntimeException(String.format("Failed to import template into encrypted RBD image %s", imageSpec), ex); + } finally { + deleteQuietly(conf); + deleteQuietly(keyring); + } + } + + private static void deleteQuietly(Path p) { + if (p == null) { + return; + } + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort cleanup of the temporary ceph auth files + } + } + /** * Best-effort probe that the local rbd CLI supports the encryption subcommand. */ From fc6019731f94c75ede96b6c54cfc9851b5109106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Tue, 7 Jul 2026 10:38:16 +0200 Subject: [PATCH 07/10] kvm: harden and align librbd-encrypted RBD volume code Review pass over the librbd LUKS2 encryption feature to fix latent issues and bring it in line with CloudStack conventions: - RbdEncryption: reject empty/null passphrase with a clear error; round rbd --size up to MiB so a non-aligned request never shrinks the volume below what was asked for; create the temporary cephx conf/keyring 0600 explicitly instead of relying on the umask. - LibvirtStorageAdaptor: close Rados/IoCTX/RbdImage in a finally block on the encrypted-root paths (mirrors deleteVolume) so handles are not leaked on exceptions; use parameterized log messages instead of string concatenation; extract the encrypted-root Option A/B logic into createEncryptedRootCoWClone / createEncryptedRootFullCopy. - RbdEncryption: use an instance logger (matching the plugin convention) and split argv construction into build{Format,Resize,Convert}Script so the generated commands can be unit-tested. --- .../kvm/storage/LibvirtStorageAdaptor.java | 184 ++++++++++-------- .../cloudstack/utils/rbd/RbdEncryption.java | 171 +++++++++------- 2 files changed, 212 insertions(+), 143 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index e2761bbffa2c..b53f26c6196c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -1370,85 +1370,10 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, boolean sameClusterRbd = srcPool.getType() == StoragePoolType.RBD && srcPool.getSourceHost().equals(destPool.getSourceHost()) && srcPool.getSourceDir().equals(destPool.getSourceDir()); - if (sameClusterRbd) { - /* - * Option A (thin CoW encrypted root). Per the Ceph "Image Encryption" clone recipe: - * grow the template base to reserve LUKS2-header space, snapshot+protect that grown state, - * clone from it, apply a LUKS2 header, then resize the clone to the requested size. The - * inherited (plaintext) template data stays readable through the clone's encryption, and - * the clone is a thin CoW image (only the header is written, not the OS data). - */ - String encSnap = rbdTemplateSnapName + "-luks"; - try { - Rados r = new Rados(destPool.getAuthUserName()); - r.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); - r.confSet("key", destPool.getAuthSecret()); - r.confSet("client_mount_timeout", "30"); - r.connect(); - IoCTX io = r.ioCtxCreate(destPool.getSourceDir()); - Rbd rbd = new Rbd(io); - RbdImage base = rbd.open(template.getName()); - boolean haveEncSnap = false; - for (RbdSnapInfo s : base.snapList()) { - if (encSnap.equals(s.name)) { - haveEncSnap = true; - break; - } - } - if (!haveEncSnap) { - base.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); - base.snapCreate(encSnap); - base.snapProtect(encSnap); - logger.debug("Prepared LUKS-reserved template snapshot " + template.getName() + "@" + encSnap); - } - rbd.close(base); - rbd.clone(template.getName(), encSnap, io, newUuid, RBD_FEATURES, rbdOrder); - r.ioCtxDestroy(io); - } catch (RadosException | RbdException e) { - logger.error("Failed to create encrypted CoW clone " + newUuid + ": " + e.getMessage()); - return null; - } - formatRbdImageEncryption(destPool, newUuid, passphrase); - if (disk.getVirtualSize() > template.getVirtualSize()) { - // grow the clone to the requested root size (encryption-aware) - new RbdEncryption().resize(destPool.getSourceHost(), destPool.getSourcePort(), - destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), - newUuid, disk.getVirtualSize(), false, passphrase); - } - disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); - return disk; + return createEncryptedRootCoWClone(template, destPool, newUuid, disk, passphrase); } - - /* - * Option B (full-copy encrypted root) for templates not on the same RBD cluster (e.g. first - * use from secondary storage). Create an empty image, apply a LUKS2 header, then import the - * template THROUGH the encryption layer (qemu-img convert -n). Correct but not thin (no CoW). - */ - long createSize = disk.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES; - try { - Rados r = new Rados(destPool.getAuthUserName()); - r.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); - r.confSet("key", destPool.getAuthSecret()); - r.confSet("client_mount_timeout", "30"); - r.connect(); - IoCTX io = r.ioCtxCreate(destPool.getSourceDir()); - Rbd rbd = new Rbd(io); - rbd.create(newUuid, createSize, RBD_FEATURES, rbdOrder); - r.ioCtxDestroy(io); - } catch (RadosException | RbdException e) { - logger.error("Failed to create encrypted RBD image " + newUuid + ": " + e.getMessage()); - return null; - } - formatRbdImageEncryption(destPool, newUuid, passphrase); - boolean srcIsRbd = srcPool.getType() == StoragePoolType.RBD; - new RbdEncryption().importTemplate( - srcIsRbd ? srcPool.getSourceDir() : null, srcIsRbd ? template.getName() : null, - srcIsRbd ? null : template.getPath(), srcIsRbd ? null : template.getFormat().toString(), - destPool.getSourceHost(), destPool.getSourcePort(), destPool.getAuthUserName(), destPool.getAuthSecret(), - destPool.getSourceDir(), newUuid, passphrase, CryptSetup.LuksType.LUKS2); - disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); - return disk; + return createEncryptedRootFullCopy(srcPool, template, destPool, newUuid, disk, passphrase); } QemuImgFile srcFile; @@ -1599,6 +1524,111 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, return disk; } + /** + * Option A (thin CoW encrypted root), used when the template already lives on the same RBD cluster + * as the destination pool. Per the Ceph "Image Encryption" clone recipe: grow the template base to + * reserve LUKS2-header space, snapshot+protect that grown state, clone from it, apply a LUKS2 header, + * then resize the clone to the requested size. The inherited (plaintext) template data stays readable + * through the clone's encryption, and the clone is a thin CoW image (only the header is written). + * + * @return the encrypted CoW clone, or {@code null} if the Ceph operations failed + */ + private KVMPhysicalDisk createEncryptedRootCoWClone(KVMPhysicalDisk template, KVMStoragePool destPool, + String newUuid, KVMPhysicalDisk disk, byte[] passphrase) { + String encSnap = rbdTemplateSnapName + "-luks"; + Rados r = null; + IoCTX io = null; + Rbd rbd = null; + RbdImage base = null; + try { + r = new Rados(destPool.getAuthUserName()); + r.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); + r.confSet("key", destPool.getAuthSecret()); + r.confSet("client_mount_timeout", "30"); + r.connect(); + io = r.ioCtxCreate(destPool.getSourceDir()); + rbd = new Rbd(io); + base = rbd.open(template.getName()); + boolean haveEncSnap = false; + for (RbdSnapInfo s : base.snapList()) { + if (encSnap.equals(s.name)) { + haveEncSnap = true; + break; + } + } + if (!haveEncSnap) { + base.resize(template.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES); + base.snapCreate(encSnap); + base.snapProtect(encSnap); + logger.debug("Prepared LUKS-reserved template snapshot {}@{}", template.getName(), encSnap); + } + rbd.clone(template.getName(), encSnap, io, newUuid, RBD_FEATURES, rbdOrder); + } catch (RadosException | RbdException e) { + logger.error("Failed to create encrypted CoW clone {}: {}", newUuid, e.getMessage()); + return null; + } finally { + if (rbd != null && base != null) { + try { + rbd.close(base); + } catch (RbdException ignored) { + // best-effort close of the template handle + } + } + if (r != null && io != null) { + r.ioCtxDestroy(io); + } + } + formatRbdImageEncryption(destPool, newUuid, passphrase); + if (disk.getVirtualSize() > template.getVirtualSize()) { + // grow the clone to the requested root size (encryption-aware) + new RbdEncryption().resize(destPool.getSourceHost(), destPool.getSourcePort(), + destPool.getAuthUserName(), destPool.getAuthSecret(), destPool.getSourceDir(), + newUuid, disk.getVirtualSize(), false, passphrase); + } + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + return disk; + } + + /** + * Option B (full-copy encrypted root), used when the template is not on the same RBD cluster (e.g. first + * use from secondary storage). Create an empty image, apply a LUKS2 header, then import the template + * THROUGH the encryption layer (qemu-img convert -n). Correct but not thin (no CoW). + * + * @return the encrypted image, or {@code null} if the Ceph operations failed + */ + private KVMPhysicalDisk createEncryptedRootFullCopy(KVMStoragePool srcPool, KVMPhysicalDisk template, + KVMStoragePool destPool, String newUuid, KVMPhysicalDisk disk, byte[] passphrase) { + long createSize = disk.getVirtualSize() + LUKS2_HEADER_RESERVE_BYTES; + Rados r = null; + IoCTX io = null; + try { + r = new Rados(destPool.getAuthUserName()); + r.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); + r.confSet("key", destPool.getAuthSecret()); + r.confSet("client_mount_timeout", "30"); + r.connect(); + io = r.ioCtxCreate(destPool.getSourceDir()); + Rbd rbd = new Rbd(io); + rbd.create(newUuid, createSize, RBD_FEATURES, rbdOrder); + } catch (RadosException | RbdException e) { + logger.error("Failed to create encrypted RBD image {}: {}", newUuid, e.getMessage()); + return null; + } finally { + if (r != null && io != null) { + r.ioCtxDestroy(io); + } + } + formatRbdImageEncryption(destPool, newUuid, passphrase); + boolean srcIsRbd = srcPool.getType() == StoragePoolType.RBD; + new RbdEncryption().importTemplate( + srcIsRbd ? srcPool.getSourceDir() : null, srcIsRbd ? template.getName() : null, + srcIsRbd ? null : template.getPath(), srcIsRbd ? null : template.getFormat().toString(), + destPool.getSourceHost(), destPool.getSourcePort(), destPool.getAuthUserName(), destPool.getAuthSecret(), + destPool.getSourceDir(), newUuid, passphrase, CryptSetup.LuksType.LUKS2); + disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS2); + return disk; + } + /** * Apply native librbd LUKS encryption to an existing RBD image via the rbd CLI. * Isolated here so the CLI dependency can later be swapped for a native (JNA) librbd binding. diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java index ce574ca70449..0cb9b21fd3ca 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/rbd/RbdEncryption.java @@ -27,6 +27,8 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermissions; /** * Thin wrapper around the {@code rbd} CLI to apply native librbd LUKS encryption to an @@ -36,9 +38,12 @@ * The CLI dependency is intentionally isolated in this class so it can later be replaced * by a native librbd (JNA) binding without touching callers. rados-java (0.x) does not * expose the rbd_encryption_format API, hence the CLI for now. + * + * The command builders ({@code build*Script}) are separated from execution so the generated + * argv can be unit-tested without a live Ceph cluster (see {@code RbdEncryptionTest}). */ public class RbdEncryption { - protected static Logger LOGGER = LogManager.getLogger(RbdEncryption.class); + protected Logger logger = LogManager.getLogger(getClass()); protected String commandPath = "rbd"; protected String qemuImgPath = "qemu-img"; @@ -49,6 +54,10 @@ public RbdEncryption(String commandPath) { this.commandPath = commandPath; } + private static String monSpec(String monHost, int monPort) { + return monPort > 0 ? monHost + ":" + monPort : monHost; + } + /** * Apply a LUKS header to an existing RBD image so librbd can transparently encrypt it. *

@@ -68,35 +77,44 @@ public RbdEncryption(String commandPath) { public void format(String monHost, int monPort, String authUser, String authSecret, String cephPool, String image, byte[] passphrase, CryptSetup.LuksType luksType) { final String imageSpec = cephPool + "/" + image; + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot LUKS-format RBD image " + imageSpec + ": empty passphrase"); + } try (KeyFile passFile = new KeyFile(passphrase); KeyFile cephKeyFile = new KeyFile(authSecret == null ? null : authSecret.getBytes(StandardCharsets.UTF_8))) { - final Script script = new Script(commandPath); - script.add("encryption"); - script.add("format"); - script.add(imageSpec); - script.add(luksType.toString()); - script.add(passFile.toString()); - script.add("--mon-host"); - script.add(monPort > 0 ? monHost + ":" + monPort : monHost); - if (authUser != null) { - script.add("--id"); - script.add(authUser); - } - if (cephKeyFile.isSet()) { - script.add("--keyfile"); - script.add(cephKeyFile.toString()); - } - + final Script script = buildFormatScript(imageSpec, luksType, passFile.toString(), + monSpec(monHost, monPort), authUser, cephKeyFile.isSet() ? cephKeyFile.toString() : null); final String result = script.execute(); if (result != null) { throw new CloudRuntimeException(String.format("Failed to apply librbd %s encryption to %s: %s", luksType, imageSpec, result)); } - LOGGER.debug("Applied {} encryption to RBD image {}", luksType, imageSpec); + logger.debug("Applied {} encryption to RBD image {}", luksType, imageSpec); } catch (IOException ex) { throw new CloudRuntimeException(String.format("Failed to apply librbd %s encryption to %s", luksType, imageSpec), ex); } } + protected Script buildFormatScript(String imageSpec, CryptSetup.LuksType luksType, String passFilePath, + String monSpec, String authUser, String cephKeyFilePath) { + final Script script = new Script(commandPath); + script.add("encryption"); + script.add("format"); + script.add(imageSpec); + script.add(luksType.toString()); + script.add(passFilePath); + script.add("--mon-host"); + script.add(monSpec); + if (authUser != null) { + script.add("--id"); + script.add(authUser); + } + if (cephKeyFilePath != null) { + script.add("--keyfile"); + script.add(cephKeyFilePath); + } + return script; + } + /** * Resize an encrypted RBD image. librbd needs the passphrase so it can resize the encrypted * payload (not just the raw image) and keep the LUKS header consistent. {@code newSizeBytes} is @@ -107,40 +125,50 @@ public void format(String monHost, int monPort, String authUser, String authSecr public void resize(String monHost, int monPort, String authUser, String authSecret, String cephPool, String image, long newSizeBytes, boolean allowShrink, byte[] passphrase) { final String imageSpec = cephPool + "/" + image; - final long sizeMiB = newSizeBytes / (1024L * 1024L); + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot resize encrypted RBD image " + imageSpec + ": empty passphrase"); + } + // rbd --size is in MiB; round up so a non-MiB-aligned request never shrinks the volume below what was asked for. + final long sizeMiB = (newSizeBytes + (1024L * 1024L) - 1) / (1024L * 1024L); try (KeyFile passFile = new KeyFile(passphrase); KeyFile cephKeyFile = new KeyFile(authSecret == null ? null : authSecret.getBytes(StandardCharsets.UTF_8))) { - final Script script = new Script(commandPath); - script.add("resize"); - script.add("--size"); - script.add(String.valueOf(sizeMiB)); - script.add(imageSpec); - script.add("--encryption-passphrase-file"); - script.add(passFile.toString()); - if (allowShrink) { - script.add("--allow-shrink"); - } - script.add("--mon-host"); - script.add(monPort > 0 ? monHost + ":" + monPort : monHost); - if (authUser != null) { - script.add("--id"); - script.add(authUser); - } - if (cephKeyFile.isSet()) { - script.add("--keyfile"); - script.add(cephKeyFile.toString()); - } - + final Script script = buildResizeScript(imageSpec, sizeMiB, passFile.toString(), allowShrink, + monSpec(monHost, monPort), authUser, cephKeyFile.isSet() ? cephKeyFile.toString() : null); final String result = script.execute(); if (result != null) { throw new CloudRuntimeException(String.format("Failed to resize encrypted RBD image %s to %d MiB: %s", imageSpec, sizeMiB, result)); } - LOGGER.debug("Resized encrypted RBD image {} to {} MiB", imageSpec, sizeMiB); + logger.debug("Resized encrypted RBD image {} to {} MiB", imageSpec, sizeMiB); } catch (IOException ex) { throw new CloudRuntimeException(String.format("Failed to resize encrypted RBD image %s", imageSpec), ex); } } + protected Script buildResizeScript(String imageSpec, long sizeMiB, String passFilePath, boolean allowShrink, + String monSpec, String authUser, String cephKeyFilePath) { + final Script script = new Script(commandPath); + script.add("resize"); + script.add("--size"); + script.add(String.valueOf(sizeMiB)); + script.add(imageSpec); + script.add("--encryption-passphrase-file"); + script.add(passFilePath); + if (allowShrink) { + script.add("--allow-shrink"); + } + script.add("--mon-host"); + script.add(monSpec); + if (authUser != null) { + script.add("--id"); + script.add(authUser); + } + if (cephKeyFilePath != null) { + script.add("--keyfile"); + script.add(cephKeyFilePath); + } + return script; + } + /** * Import a template into an already-created, already-LUKS-formatted RBD image by writing it * THROUGH the librbd encryption layer with {@code qemu-img convert -n} (so the data lands @@ -157,39 +185,26 @@ public void importTemplate(String srcRbdPool, String srcRbdImage, String monHost, int monPort, String authUser, String authSecret, String cephPool, String destImage, byte[] passphrase, CryptSetup.LuksType luksType) { final String imageSpec = cephPool + "/" + destImage; - final String monSpec = monPort > 0 ? monHost + ":" + monPort : monHost; + if (passphrase == null || passphrase.length == 0) { + throw new CloudRuntimeException("Cannot import template into encrypted RBD image " + imageSpec + ": empty passphrase"); + } Path conf = null; Path keyring = null; try (KeyFile passFile = new KeyFile(passphrase)) { - keyring = Files.createTempFile("cs-ceph-", ".keyring"); // 0600 by default on POSIX + // These temp files hold the cephx secret; create them 0600 up front (matching KeyFile) rather than relying on the umask. + final FileAttribute ownerOnly = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------")); + keyring = Files.createTempFile("cs-ceph-", ".keyring", ownerOnly); Files.writeString(keyring, "[client." + authUser + "]\n\tkey = " + authSecret + "\n"); - conf = Files.createTempFile("cs-ceph-", ".conf"); - Files.writeString(conf, "[global]\nmon_host = " + monSpec + "\nkeyring = " + keyring + "\n"); - - final Script q = new Script(qemuImgPath); - q.add("convert"); - q.add("-n"); // target already exists (pre-created + luks-formatted) - if (srcRbdImage != null) { - q.add("--image-opts"); - q.add("driver=rbd,pool=" + srcRbdPool + ",image=" + srcRbdImage + ",conf=" + conf + ",user=" + authUser); - } else { - if (srcFileFormat != null) { - q.add("-f"); - q.add(srcFileFormat.toLowerCase()); - } - q.add(srcFilePath); - } - q.add("--object"); - q.add("secret,id=luks0,file=" + passFile.toString()); - q.add("--target-image-opts"); - q.add("driver=rbd,pool=" + cephPool + ",image=" + destImage + ",conf=" + conf + ",user=" + authUser - + ",encrypt.format=" + luksType + ",encrypt.key-secret=luks0"); + conf = Files.createTempFile("cs-ceph-", ".conf", ownerOnly); + Files.writeString(conf, "[global]\nmon_host = " + monSpec(monHost, monPort) + "\nkeyring = " + keyring + "\n"); + final Script q = buildConvertScript(srcRbdPool, srcRbdImage, srcFilePath, srcFileFormat, + cephPool, destImage, conf.toString(), authUser, passFile.toString(), luksType); final String result = q.execute(); if (result != null) { throw new CloudRuntimeException(String.format("Failed to import template into encrypted RBD image %s: %s", imageSpec, result)); } - LOGGER.debug("Imported template into encrypted RBD image {}", imageSpec); + logger.debug("Imported template into encrypted RBD image {}", imageSpec); } catch (IOException ex) { throw new CloudRuntimeException(String.format("Failed to import template into encrypted RBD image %s", imageSpec), ex); } finally { @@ -198,6 +213,30 @@ public void importTemplate(String srcRbdPool, String srcRbdImage, } } + protected Script buildConvertScript(String srcRbdPool, String srcRbdImage, String srcFilePath, String srcFileFormat, + String cephPool, String destImage, String confPath, String authUser, + String passFilePath, CryptSetup.LuksType luksType) { + final Script q = new Script(qemuImgPath); + q.add("convert"); + q.add("-n"); // target already exists (pre-created + luks-formatted) + if (srcRbdImage != null) { + q.add("--image-opts"); + q.add("driver=rbd,pool=" + srcRbdPool + ",image=" + srcRbdImage + ",conf=" + confPath + ",user=" + authUser); + } else { + if (srcFileFormat != null) { + q.add("-f"); + q.add(srcFileFormat.toLowerCase()); + } + q.add(srcFilePath); + } + q.add("--object"); + q.add("secret,id=luks0,file=" + passFilePath); + q.add("--target-image-opts"); + q.add("driver=rbd,pool=" + cephPool + ",image=" + destImage + ",conf=" + confPath + ",user=" + authUser + + ",encrypt.format=" + luksType + ",encrypt.key-secret=luks0"); + return q; + } + private static void deleteQuietly(Path p) { if (p == null) { return; From ef726a0121a639173170ba401709905ed9f16d97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Tue, 7 Jul 2026 10:38:16 +0200 Subject: [PATCH 08/10] kvm: add RbdEncryption unit tests Assert the rbd/qemu-img argv built for format, resize and convert-through-encryption (RBD and file sources), and that empty/null passphrases are rejected. Command construction is verified without a live Ceph cluster. --- .../utils/rbd/RbdEncryptionTest.java | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java new file mode 100644 index 000000000000..d31c63d2197e --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/rbd/RbdEncryptionTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.utils.rbd; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.cryptsetup.CryptSetup; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import java.nio.charset.StandardCharsets; + +/** + * Unit tests for {@link RbdEncryption}. These assert the {@code rbd}/{@code qemu-img} argv that + * would be handed to {@link Script}, so the command construction is verified without a live Ceph + * cluster (the actual execution needs a real cluster and is covered by end-to-end testing). + */ +@RunWith(MockitoJUnitRunner.class) +public class RbdEncryptionTest { + + private final RbdEncryption rbdEncryption = new RbdEncryption(); + + @Test + public void buildFormatScriptWithCephxAuth() { + Script script = rbdEncryption.buildFormatScript("cloudstack/img", CryptSetup.LuksType.LUKS2, + "/tmp/pass", "1.2.3.4:6789", "cloudstack", "/tmp/key"); + String cmd = script.toString(); + Assert.assertTrue(cmd, cmd.contains("rbd encryption format cloudstack/img luks2 /tmp/pass")); + Assert.assertTrue(cmd, cmd.contains("--mon-host 1.2.3.4:6789")); + Assert.assertTrue(cmd, cmd.contains("--id cloudstack")); + Assert.assertTrue(cmd, cmd.contains("--keyfile /tmp/key")); + } + + @Test + public void buildFormatScriptWithoutCephxAuth() { + // authUser == null and cephKeyFilePath == null (e.g. auth-less cluster): no --id / --keyfile. + Script script = rbdEncryption.buildFormatScript("pool/vol", CryptSetup.LuksType.LUKS2, + "/tmp/pass", "mon:6789", null, null); + String cmd = script.toString(); + Assert.assertTrue(cmd, cmd.contains("rbd encryption format pool/vol luks2 /tmp/pass --mon-host mon:6789")); + Assert.assertFalse(cmd, cmd.contains("--id")); + Assert.assertFalse(cmd, cmd.contains("--keyfile")); + } + + @Test + public void buildResizeScriptGrowDoesNotAllowShrink() { + Script script = rbdEncryption.buildResizeScript("cloudstack/img", 10240L, "/tmp/pass", false, + "1.2.3.4:6789", "cloudstack", "/tmp/key"); + String cmd = script.toString(); + Assert.assertTrue(cmd, cmd.contains("rbd resize --size 10240 cloudstack/img --encryption-passphrase-file /tmp/pass")); + Assert.assertTrue(cmd, cmd.contains("--id cloudstack")); + Assert.assertFalse(cmd, cmd.contains("--allow-shrink")); + } + + @Test + public void buildResizeScriptShrinkPassesAllowShrink() { + Script script = rbdEncryption.buildResizeScript("cloudstack/img", 5120L, "/tmp/pass", true, + "1.2.3.4:6789", "cloudstack", "/tmp/key"); + Assert.assertTrue(script.toString(), script.toString().contains("--allow-shrink")); + } + + @Test + public void buildConvertScriptFromRbdSource() { + Script q = rbdEncryption.buildConvertScript("srcpool", "srcimg", null, null, + "cloudstack", "dst", "/tmp/conf", "cloudstack", "/tmp/pass", CryptSetup.LuksType.LUKS2); + String cmd = q.toString(); + Assert.assertTrue(cmd, cmd.contains("qemu-img convert -n")); + Assert.assertTrue(cmd, cmd.contains("--image-opts driver=rbd,pool=srcpool,image=srcimg,conf=/tmp/conf,user=cloudstack")); + Assert.assertTrue(cmd, cmd.contains("--object secret,id=luks0,file=/tmp/pass")); + Assert.assertTrue(cmd, cmd.contains("--target-image-opts driver=rbd,pool=cloudstack,image=dst,conf=/tmp/conf,user=cloudstack,encrypt.format=luks2,encrypt.key-secret=luks0")); + } + + @Test + public void buildConvertScriptFromFileSource() { + Script q = rbdEncryption.buildConvertScript(null, null, "/tmp/tmpl.qcow2", "QCOW2", + "cloudstack", "dst", "/tmp/conf", "cloudstack", "/tmp/pass", CryptSetup.LuksType.LUKS2); + String cmd = q.toString(); + Assert.assertTrue(cmd, cmd.contains("-f qcow2 /tmp/tmpl.qcow2")); + Assert.assertFalse(cmd, cmd.contains("--image-opts")); + Assert.assertTrue(cmd, cmd.contains("encrypt.format=luks2,encrypt.key-secret=luks0")); + } + + @Test + public void formatRejectsEmptyPassphrase() { + Assert.assertThrows(CloudRuntimeException.class, () -> rbdEncryption.format( + "1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "img", + new byte[0], CryptSetup.LuksType.LUKS2)); + } + + @Test + public void resizeRejectsNullPassphrase() { + Assert.assertThrows(CloudRuntimeException.class, () -> rbdEncryption.resize( + "1.2.3.4", 6789, "cloudstack", "secret", "cloudstack", "img", + 1L << 30, false, null)); + } + + @Test + public void importTemplateRejectsEmptyPassphrase() { + Assert.assertThrows(CloudRuntimeException.class, () -> rbdEncryption.importTemplate( + "srcpool", "srcimg", null, null, "1.2.3.4", 6789, "cloudstack", "secret", + "cloudstack", "dst", "".getBytes(StandardCharsets.UTF_8), CryptSetup.LuksType.LUKS2)); + } +} From a17b777e56a5fe01e6752879445c33a8901e2c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Tue, 7 Jul 2026 11:40:04 +0200 Subject: [PATCH 09/10] kvm: refuse encrypted RBD hot-plug on libvirt < 10.1.0 libvirt 10.0.0 has an object apply-order bug (fixed in 10.1.0) that breaks hot-plug of an encrypted rbd blockdev: on attach the disk is opened before its LUKS secret object is defined, so the attach fails with "No secret with id '...-format-encryption-secret0'". Booting a VM from an encrypted RBD disk is unaffected (the QEMU command line resolves all -object before -blockdev). Refuse the attach up front with a clear error (mirroring the existing openvswitch/io_uring libvirt-version gates) instead of letting libvirt fail opaquely. Only the RBD hot-plug path is gated; boot/root/detach are untouched. --- .../kvm/storage/KVMStorageProcessor.java | 23 +++++++++++++++++++ .../kvm/storage/KVMStorageProcessorTest.java | 22 ++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index ba820a3a6c58..5727db5769cd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -181,6 +181,12 @@ public class KVMStorageProcessor implements StorageProcessor { private static final String CEPH_AUTH_KEY = "key"; private static final String CEPH_CLIENT_MOUNT_TIMEOUT = "client_mount_timeout"; private static final String CEPH_DEFAULT_MOUNT_TIMEOUT = "30"; + + // libvirt < 10.1.0 has an object apply-order bug (fixed in 10.1.0) that breaks hot-plug of an encrypted + // blockdev: on attach the disk is opened before its LUKS secret object is defined, so the attach fails with + // "No secret with id '...-format-encryption-secret0'". Booting a VM from an encrypted disk is unaffected (the + // QEMU command line resolves all -object before -blockdev). See qemuBlockStorageSourceAttachApply() in libvirt. + private static final long MIN_LIBVIRT_VERSION_FOR_RBD_ENCRYPTED_HOTPLUG = 10001000L; // libvirt 10.1.0 /** * Time interval before rechecking virsh commands */ @@ -1789,6 +1795,20 @@ protected DiskDef.DiskBus getAttachDiskBusType(int deviceId, List disks return DiskDef.DiskBus.VIRTIO; } + /** + * libvirt < 10.1.0 cannot hot-plug an encrypted rbd blockdev (the LUKS secret is applied after the disk is + * opened), so refuse the attach with a clear message rather than letting libvirt fail with an opaque + * "No secret with id ..." error. Only the RBD hot-plug path is affected; booting a VM from an encrypted RBD + * disk works on older libvirt, so this does not gate the boot/root path. + */ + protected void ensureLibvirtSupportsEncryptedRbdHotplug(StoragePoolType poolType) { + if (poolType == StoragePoolType.RBD + && resource.getHypervisorLibvirtVersion() < MIN_LIBVIRT_VERSION_FOR_RBD_ENCRYPTED_HOTPLUG) { + throw new CloudRuntimeException("Libvirt version 10.1.0 required to attach an encrypted RBD volume to a running VM, but version " + + resource.getHypervisorLibvirtVersion() + " detected. Booting a VM from an encrypted RBD disk is not affected."); + } + } + @Override public Answer attachVolume(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); @@ -1801,6 +1821,9 @@ public Answer attachVolume(final AttachCommand cmd) { final Connect conn = LibvirtConnection.getConnectionByVmName(vmName); DiskDef.LibvirtDiskEncryptDetails encryptDetails = null; if (vol.requiresEncryption()) { + // Encrypted RBD is decrypted by librbd inside qemu; hot-plugging it needs a libvirt new enough to + // emit the LUKS secret before the rbd blockdev. Booting from an encrypted RBD disk is unaffected. + ensureLibvirtSupportsEncryptedRbdHotplug(primaryStore.getPoolType()); String secretUuid = resource.createLibvirtVolumeSecret(conn, vol.getPath(), vol.getPassphrase()); // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine. String encryptEngine = (primaryStore.getPoolType() == StoragePoolType.RBD) ? "librbd" : null; diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java index 0a58dfc79b0d..b895aa65ff59 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java @@ -21,6 +21,7 @@ import com.cloud.exception.InternalErrorException; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.storage.Storage; import com.cloud.storage.template.TemplateConstants; @@ -666,4 +667,25 @@ public void testComputeMd5Hash_ConsistentResults() { Assert.fail("Failed to test computeMd5Hash: " + e.getMessage()); } } + + @Test + public void ensureLibvirtSupportsEncryptedRbdHotplugRejectsOldLibvirt() { + Mockito.when(resource.getHypervisorLibvirtVersion()).thenReturn(10000000L); // libvirt 10.0.0 + CloudRuntimeException ex = Assert.assertThrows(CloudRuntimeException.class, + () -> storageProcessor.ensureLibvirtSupportsEncryptedRbdHotplug(StoragePoolType.RBD)); + Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("Libvirt version 10.1.0 required")); + } + + @Test + public void ensureLibvirtSupportsEncryptedRbdHotplugAllowsNewLibvirt() { + Mockito.when(resource.getHypervisorLibvirtVersion()).thenReturn(10001000L); // libvirt 10.1.0 + storageProcessor.ensureLibvirtSupportsEncryptedRbdHotplug(StoragePoolType.RBD); // must not throw + } + + @Test + public void ensureLibvirtSupportsEncryptedRbdHotplugIgnoresNonRbd() { + // The libvirt hot-plug apply-order bug is RBD-specific; other pool types are not gated, and the libvirt + // version is not even consulted for them. + storageProcessor.ensureLibvirtSupportsEncryptedRbdHotplug(StoragePoolType.NetworkFilesystem); + } } From ad357f5ca31d02b5aafad133351da597dfa559d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Rozsyp=C3=A1lek?= Date: Tue, 7 Jul 2026 11:59:26 +0200 Subject: [PATCH 10/10] docs: add PendingReleaseNotes entry for librbd-encrypted RBD volumes --- PendingReleaseNotes | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 9670b6e7c13a..76c040af002f 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -39,3 +39,13 @@ example.ver.1 > example.ver.2: which can now be attached to Instances. This is to prevent the Secondary Storage to grow to enormous sizes as Linux Distributions keep growing in size while a stripped down Linux should fit on a 2.88MB floppy. + + +4.22.0.0 > 4.23.0.0: + * KVM/Ceph: RBD volumes can now be encrypted at rest using native librbd + LUKS2 (), for both data disks + and root disks. Encryption is transparent to the guest and reuses the + existing CloudStack volume-encryption passphrase handling, so no + additional key store is required. Note: attaching an encrypted RBD volume + to a running Instance requires libvirt >= 10.1.0; booting an Instance from + an encrypted RBD root disk works on older libvirt.