storage: KVM - enable RBD/Ceph volume encryption support

PendingReleaseNotes

@@ -39,3 +39,13 @@ example.ver.1 > example.ver.2:
     which can now be attached to Instances. This is to prevent the Secondary
     Storage to grow to enormous sizes as Linux Distributions keep growing in
     size while a stripped down Linux should fit on a 2.88MB floppy.
+
+
+4.22.0.0 > 4.23.0.0:
+  * KVM/Ceph: RBD volumes can now be encrypted at rest using native librbd
+    LUKS2 (<encryption format='luks2' engine='librbd'>), for both data disks
+    and root disks. Encryption is transparent to the guest and reuses the
+    existing CloudStack volume-encryption passphrase handling, so no
+    additional key store is required. Note: attaching an encrypted RBD volume
+    to a running Instance requires libvirt >= 10.1.0; booting an Instance from
+    an encrypted RBD root disk works on older libvirt.

api/src/main/java/com/cloud/storage/Storage.java

@@ -171,7 +171,7 @@ public static enum StoragePoolType {
         LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR
         CLVM(true, false, EncryptionSupport.Unsupported),
         CLVM_NG(true, false, EncryptionSupport.Hypervisor),
-        RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD
+        RBD(true, true, EncryptionSupport.Hypervisor), // http://libvirt.org/storage.html#StorageBackendRBD ; encrypted natively by librbd (LUKS2, engine='librbd')
         SharedMountPoint(true, true, EncryptionSupport.Hypervisor),
         VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage
         PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers.

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -89,6 +89,7 @@
 import org.apache.cloudstack.storage.volume.VolumeOnStorageTO;
 import org.apache.cloudstack.utils.bytescale.ByteScaleUtils;
 import org.apache.cloudstack.utils.cryptsetup.CryptSetup;
+import org.apache.cloudstack.utils.rbd.RbdEncryption;
 import org.apache.cloudstack.utils.hypervisor.HypervisorUtils;
 import org.apache.cloudstack.utils.linux.CPUStat;
 import org.apache.cloudstack.utils.linux.KVMHostInfo;
@@ -3864,7 +3865,9 @@ public int compare(final DiskTO arg0, final DiskTO arg1) {
                 if (volumeObjectTO.requiresEncryption() &&
                         pool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor ) {
                     String secretUuid = createLibvirtVolumeSecret(conn, volumeObjectTO.getPath(), volumeObjectTO.getPassphrase());
-                    DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat()));
+                    // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine.
+                    String encryptEngine = (pool.getType() == StoragePoolType.RBD) ? "librbd" : null;
+                    DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat()), encryptEngine);
                     disk.setLibvirtDiskEncryptDetails(encryptDetails);
                 }
             }
@@ -6164,10 +6167,28 @@ public boolean isHostSecured() {
     }
 
     /**
-     * Test host for volume encryption support
+     * Test host for volume encryption support. A host is considered encryption-capable if it
+     * supports EITHER mechanism CloudStack can use:
+     *  - qemu-native LUKS (qemu-img LUKS + cryptsetup) for file/block backed pools, or
+     *  - librbd native encryption (rbd encryption format) for RBD/Ceph pools.
+     * NOTE: HOST_VOLUME_ENCRYPTION is a single host-wide flag and is not per-pool, so a host that
+     * advertises encryption via only one mechanism could still be selected for a volume that needs
+     * the other. In practice hosts that do encryption have the qemu-native stack; the librbd branch
+     * additionally covers Ceph-only hosts.
      * @return boolean
      */
     public boolean hostSupportsVolumeEncryption() {
+        boolean supported = hostSupportsQemuNativeVolumeEncryption() || hostSupportsRbdVolumeEncryption();
+        if (!supported) {
+            LOGGER.info("Host does not support volume encryption (no qemu-native LUKS + cryptsetup, and no librbd rbd encryption)");
+        }
+        return supported;
+    }
+
+    /**
+     * Test host for qemu-native LUKS volume encryption (qemu-img LUKS support + cryptsetup).
+     */
+    public boolean hostSupportsQemuNativeVolumeEncryption() {
         // test qemu-img
         try {
             QemuImg qemu = new QemuImg(0);
@@ -6189,6 +6210,13 @@ public boolean hostSupportsVolumeEncryption() {
         return true;
     }
 
+    /**
+     * Test host for librbd native LUKS encryption support (rbd CLI with the encryption subcommand).
+     */
+    public boolean hostSupportsRbdVolumeEncryption() {
+        return new RbdEncryption().isSupported();
+    }
+
     public boolean isSecureMode(String bootMode) {
         if (StringUtils.isNotBlank(bootMode) && "secure".equalsIgnoreCase(bootMode)) {
             return true;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java

@@ -788,14 +788,21 @@ public static class DiskDef {
         public static class LibvirtDiskEncryptDetails {
             String passphraseUuid;
             QemuObject.EncryptFormat encryptFormat;
+            String engine; // optional libvirt encryption engine (e.g. "librbd"); null => libvirt/qemu default
 
             public LibvirtDiskEncryptDetails(String passphraseUuid, QemuObject.EncryptFormat encryptFormat) {
+                this(passphraseUuid, encryptFormat, null);
+            }
+
+            public LibvirtDiskEncryptDetails(String passphraseUuid, QemuObject.EncryptFormat encryptFormat, String engine) {
                 this.passphraseUuid = passphraseUuid;
                 this.encryptFormat = encryptFormat;
+                this.engine = engine;
             }
 
             public String getPassphraseUuid() { return this.passphraseUuid; }
             public QemuObject.EncryptFormat getEncryptFormat() { return this.encryptFormat; }
+            public String getEngine() { return this.engine; }
         }
 
         public static class DiskGeometry {
@@ -1437,7 +1444,11 @@ public String toString() {
             }
 
             if (encryptDetails != null) {
-                diskBuilder.append("<encryption format='" + encryptDetails.encryptFormat + "'>\n");
+                diskBuilder.append("<encryption format='" + encryptDetails.encryptFormat + "'");
+                if (encryptDetails.engine != null) {
+                    diskBuilder.append(" engine='" + encryptDetails.engine + "'");
+                }
+                diskBuilder.append(">\n");
                 diskBuilder.append("<secret type='passphrase' uuid='" + encryptDetails.passphraseUuid + "' />\n");
                 diskBuilder.append("</encryption>\n");
             }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java

@@ -33,6 +33,7 @@
 import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
 import org.apache.cloudstack.utils.qemu.QemuImgException;
 import org.apache.cloudstack.utils.qemu.QemuObject;
+import org.apache.cloudstack.utils.rbd.RbdEncryption;
 import org.libvirt.Connect;
 import org.libvirt.Domain;
 import org.libvirt.DomainInfo;
@@ -93,6 +94,13 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR
             final String path = vol.getPath();
             String type = notifyOnlyType;
 
+            // Encrypted RBD volumes are encrypted natively by librbd; they must be resized via
+            // `rbd resize --encryption-passphrase-file` so librbd grows the encrypted payload and keeps
+            // the LUKS header consistent. The libvirt/qemu-img resize paths below are for qemu-native
+            // encryption and would not handle the librbd LUKS2 layout.
+            final boolean rbdEncrypted = pool.getType() == StoragePoolType.RBD
+                    && command.getPassphrase() != null && command.getPassphrase().length > 0;
+
             if (spool.getType().equals(StoragePoolType.PowerFlex) && vol.getFormat().equals(PhysicalDiskFormat.QCOW2)) {
                 // PowerFlex QCOW2 sizing needs to consider overhead.
                 newSize = ScaleIOStorageAdaptor.getUsableBytesFromRawBytes(newSize);
@@ -115,7 +123,7 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR
             /* libvirt doesn't support resizing (C)LVM devices, and corrupts QCOW2 in some scenarios, so we have to do these via qemu-img */
             if (pool.getType() != StoragePoolType.CLVM && pool.getType() != StoragePoolType.CLVM_NG
                     && pool.getType() != StoragePoolType.Linstor && pool.getType() != StoragePoolType.PowerFlex
-                    && vol.getFormat() != PhysicalDiskFormat.QCOW2) {
+                    && vol.getFormat() != PhysicalDiskFormat.QCOW2 && !rbdEncrypted) {
                 logger.debug("Volume " + path +  " can be resized by libvirt. Asking libvirt to resize the volume.");
                 try {
                     final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
@@ -139,11 +147,15 @@ public Answer execute(final ResizeVolumeCommand command, final LibvirtComputingR
 
             boolean vmIsRunning = isVmRunning(vmInstanceName, libvirtComputingResource);
 
-            /* when VM is offline, we use qemu-img directly to resize encrypted volumes.
-               If VM is online, the existing resize script will call virsh blockresize which works
-               with both encrypted and non-encrypted volumes.
+            /* when VM is offline, we use qemu-img (or rbd, for librbd-encrypted RBD) directly to resize
+               encrypted volumes. If VM is online, the existing resize script calls virsh blockresize,
+               which for an librbd-encrypted RBD disk lets qemu/librbd grow the encrypted payload and
+               notify the guest in one step (no passphrase needed, qemu already has the secret loaded).
              */
-            if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) {
+            if (rbdEncrypted && !vmIsRunning) {
+                logger.debug("Invoking rbd to resize an offline, encrypted (librbd) RBD volume");
+                resizeRbdEncryptedVolume(pool, vol, newSize, shrinkOk, command.getPassphrase());
+            } else if (!vmIsRunning && command.getPassphrase() != null && command.getPassphrase().length > 0 ) {
                 logger.debug("Invoking qemu-img to resize an offline, encrypted volume");
                 QemuObject.EncryptFormat encryptFormat = QemuObject.EncryptFormat.enumValue(command.getEncryptFormat());
                 resizeEncryptedQcowFile(vol, encryptFormat,newSize, command.getPassphrase(), libvirtComputingResource);
@@ -213,6 +225,16 @@ private void resizeEncryptedQcowFile(final KVMPhysicalDisk vol, final QemuObject
         }
     }
 
+    private void resizeRbdEncryptedVolume(final KVMStoragePool pool, final KVMPhysicalDisk vol, long newSize,
+                                          boolean shrinkOk, byte[] passphrase) throws CloudRuntimeException {
+        try {
+            new RbdEncryption().resize(pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(),
+                    pool.getAuthSecret(), pool.getSourceDir(), vol.getName(), newSize, shrinkOk, passphrase);
+        } finally {
+            Arrays.fill(passphrase, (byte) 0);
+        }
+    }
+
     private Answer handleMultipathSCSIResize(ResizeVolumeCommand command, KVMStoragePool pool) {
         ((MultipathSCSIPool)pool).resize(command.getPath(), command.getInstanceName(), command.getNewSize());
         return new ResizeVolumeAnswer(command, true, "");

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -181,6 +181,12 @@ public class KVMStorageProcessor implements StorageProcessor {
     private static final String CEPH_AUTH_KEY = "key";
     private static final String CEPH_CLIENT_MOUNT_TIMEOUT = "client_mount_timeout";
     private static final String CEPH_DEFAULT_MOUNT_TIMEOUT = "30";
+
+    // libvirt < 10.1.0 has an object apply-order bug (fixed in 10.1.0) that breaks hot-plug of an encrypted
+    // blockdev: on attach the disk is opened before its LUKS secret object is defined, so the attach fails with
+    // "No secret with id '...-format-encryption-secret0'". Booting a VM from an encrypted disk is unaffected (the
+    // QEMU command line resolves all -object before -blockdev). See qemuBlockStorageSourceAttachApply() in libvirt.
+    private static final long MIN_LIBVIRT_VERSION_FOR_RBD_ENCRYPTED_HOTPLUG = 10001000L; // libvirt 10.1.0
     /**
      * Time interval before rechecking virsh commands
      */
@@ -1789,6 +1795,20 @@ protected DiskDef.DiskBus getAttachDiskBusType(int deviceId, List<DiskDef> disks
         return DiskDef.DiskBus.VIRTIO;
     }
 
+    /**
+     * libvirt &lt; 10.1.0 cannot hot-plug an encrypted rbd blockdev (the LUKS secret is applied after the disk is
+     * opened), so refuse the attach with a clear message rather than letting libvirt fail with an opaque
+     * "No secret with id ..." error. Only the RBD hot-plug path is affected; booting a VM from an encrypted RBD
+     * disk works on older libvirt, so this does not gate the boot/root path.
+     */
+    protected void ensureLibvirtSupportsEncryptedRbdHotplug(StoragePoolType poolType) {
+        if (poolType == StoragePoolType.RBD
+                && resource.getHypervisorLibvirtVersion() < MIN_LIBVIRT_VERSION_FOR_RBD_ENCRYPTED_HOTPLUG) {
+            throw new CloudRuntimeException("Libvirt version 10.1.0 required to attach an encrypted RBD volume to a running VM, but version "
+                    + resource.getHypervisorLibvirtVersion() + " detected. Booting a VM from an encrypted RBD disk is not affected.");
+        }
+    }
+
     @Override
     public Answer attachVolume(final AttachCommand cmd) {
         final DiskTO disk = cmd.getDisk();
@@ -1801,8 +1821,13 @@ public Answer attachVolume(final AttachCommand cmd) {
             final Connect conn = LibvirtConnection.getConnectionByVmName(vmName);
             DiskDef.LibvirtDiskEncryptDetails encryptDetails = null;
             if (vol.requiresEncryption()) {
+                // Encrypted RBD is decrypted by librbd inside qemu; hot-plugging it needs a libvirt new enough to
+                // emit the LUKS secret before the rbd blockdev. Booting from an encrypted RBD disk is unaffected.
+                ensureLibvirtSupportsEncryptedRbdHotplug(primaryStore.getPoolType());
                 String secretUuid = resource.createLibvirtVolumeSecret(conn, vol.getPath(), vol.getPassphrase());
-                encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(vol.getEncryptFormat()));
+                // RBD volumes are encrypted natively by librbd, so request the librbd encryption engine.
+                String encryptEngine = (primaryStore.getPoolType() == StoragePoolType.RBD) ? "librbd" : null;
+                encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(vol.getEncryptFormat()), encryptEngine);
                 vol.clearPassphrase();
             }