KVM: live volume migration on Ceph RBD, and between RBD, NFS, SharedMountPoint and local pools

core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java

@@ -124,6 +124,14 @@ public DataTO getDestData() {
         return destData;
     }
 
+    public void setSrcData(DataTO srcData) {
+        this.srcData = srcData;
+    }
+
+    public void setDestData(DataTO destData) {
+        this.destData = destData;
+    }
+
     public void setSrcDetails(Map<String, String> details) {
         srcDetails = details;
     }

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -642,6 +642,8 @@ protected Answer migrateVolumeToPool(DataObject srcData, DataObject destData) {
             command.setContextParam(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString());
         }
 
+        command.setSrcData(srcData.getTO());
+        command.setDestData(destData.getTO());
         boolean encryptionRequired = anyVolumeRequiresEncryption(srcData, destData);
 
         EndPoint ep = selector.select(srcData, StorageAction.MIGRATEVOLUME);
@@ -675,6 +677,23 @@ protected Answer migrateVolumeToPool(DataObject srcData, DataObject destData) {
             volumeVo.setLastPoolId(oldPoolId);
             if (destPool.getPoolType() == StoragePoolType.CLVM) {
                 volumeVo.setFormat(ImageFormat.RAW);
+            } else if (destPool.getPoolType() == StoragePoolType.RBD) {
+                // RBD only ever stores raw images; keep the volume's format metadata consistent
+                // regardless of what format it had on the source pool (e.g. QCOW2 on NFS).
+                volumeVo.setFormat(ImageFormat.RAW);
+            } else if (srcPool.getPoolType() == StoragePoolType.RBD && volumeVo.getFormat() == ImageFormat.RAW
+                    && (destPool.getPoolType() == StoragePoolType.NetworkFilesystem
+                    || destPool.getPoolType() == StoragePoolType.SharedMountPoint
+                    || destPool.getPoolType() == StoragePoolType.Filesystem)) {
+                // The KVM agent (LibvirtMigrateVolumeCommandWrapper) writes a real QCOW2 file when the
+                // destination is file-based; without updating this, the volume keeps looking RAW to
+                // LibvirtComputingResource, which then (mis)treats a plain qcow2 file on NFS as a raw
+                // block device the next time the Instance is started.
+                volumeVo.setFormat(ImageFormat.QCOW2);
+            }
+            if (volumeVo.getEncryptFormat() != null && (srcPool.getPoolType() == StoragePoolType.RBD) != (destPool.getPoolType() == StoragePoolType.RBD)) {
+                // The KVM agent encrypts an RBD volume natively with librbd (LUKS2), and a volume on any other pool with qemu (LUKS)
+                volumeVo.setEncryptFormat(destPool.getPoolType() == StoragePoolType.RBD ? "luks2" : "luks");
             }
             if (ClvmPoolManager.isClvmPoolType(destPool.getPoolType())) {
                 clvmPoolManager.setClvmLockHostId(volume.getId(), ep.getId());

engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java

@@ -603,7 +603,7 @@ public EndPoint select(DataObject object, StorageAction action, boolean encrypti
             }
             case MIGRATEVOLUME: {
                 VolumeInfo volume = (VolumeInfo) object;
-                if (volume.getHypervisorType() == Hypervisor.HypervisorType.Hyperv || volume.getHypervisorType() == Hypervisor.HypervisorType.VMware) {
+                if (List.of(Hypervisor.HypervisorType.VMware, Hypervisor.HypervisorType.Hyperv, Hypervisor.HypervisorType.KVM).contains(volume.getHypervisorType())) {
                     VirtualMachine vm = volume.getAttachedVM();
                     if ((vm != null) && (vm.getState() == VirtualMachine.State.Running)) {
                         Long hostId = vm.getHostId();

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStorageVolumeXMLParser.java

@@ -48,7 +48,7 @@ public LibvirtStorageVolumeDef parseStorageVolumeXML(String volXML) {
 
             String VolName = getTagValue("name", rootElement);
             Element target = (Element)rootElement.getElementsByTagName("target").item(0);
-            String format = getAttrValue("type", "format", target);
+            String format = getAttrValue("format", "type", target);
             Long capacity = Long.parseLong(getTagValue("capacity", rootElement));
             return new LibvirtStorageVolumeDef(VolName, capacity, LibvirtStorageVolumeDef.VolumeFormat.getFormat(format), null, null);
         } catch (ParserConfigurationException e) {

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java

@@ -169,4 +169,8 @@ default LibvirtVMDef.DiskDef.BlockIOSize getSupportedPhysicalBlockSize() {
 
     default void customizeLibvirtDiskDef(LibvirtVMDef.DiskDef disk) {
     }
+
+    default String getAuthSecretUUID() {
+        return null;
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -3488,7 +3488,8 @@ public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
             newVol.setPath(path);
             ImageFormat newVolumeFormat = getFormat(newVolume.getFormat());
             newVol.setFormat(newVolumeFormat);
-            newVol.setEncryptFormat(destVol.getEncryptFormat());
+            // an encrypted volume copied to another kind of pool is encrypted differently (librbd LUKS2 on RBD, qemu LUKS otherwise)
+            newVol.setEncryptFormat(newVolume.getQemuEncryptFormat() != null ? newVolume.getQemuEncryptFormat().toString() : destVol.getEncryptFormat());
             return new CopyCmdAnswer(newVol);
         } catch (final CloudRuntimeException e) {
             logger.debug("Failed to copyVolumeFromPrimaryToPrimary: ", e);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java

@@ -21,8 +21,12 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermissions;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.EnumMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -45,6 +49,7 @@
 import org.apache.cloudstack.utils.qemu.QemuObject;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -77,6 +82,7 @@
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageLayer;
+import com.cloud.utils.RbdUtils;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.Script;
 import com.cloud.utils.storage.TemplateDownloaderUtil;
@@ -98,10 +104,16 @@ public class LibvirtStorageAdaptor implements StorageAdaptor {
     private int rbdOrder = 0; /* Order 0 means 4MB blocks (the default) */
     /* Space reserved at the front of an encrypted RBD image for the LUKS2 header/keyslots so the
        usable (decrypted) size still matches the requested volume size. */
-    private static final long LUKS2_HEADER_RESERVE_BYTES = 16L << 20; // 16 MiB
+    public static final long LUKS2_HEADER_RESERVE_BYTES = 16L << 20; // 16 MiB
     /* libvirt's VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS, not exposed as a constant by libvirt-java */
     private static final int VIR_STORAGE_VOL_DELETE_WITH_SNAPSHOTS = 2;
 
+    /* pool types an encrypted volume can be copied between, see copyEncryptedPhysicalDisk */
+    private static final Set<StoragePoolType> ENCRYPTED_COPY_POOL_TYPES = Set.of(StoragePoolType.RBD, StoragePoolType.NetworkFilesystem,
+            StoragePoolType.Filesystem, StoragePoolType.SharedMountPoint);
+    private static final String SRC_LUKS_SECRET_ID = "srcsec0";
+    private static final String DEST_LUKS_SECRET_ID = "destsec0";
+
     private static final Set<StoragePoolType> QEMU_IMG_MANAGED_POOL_TYPES = Set.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem, StoragePoolType.SharedMountPoint);
 
     public LibvirtStorageAdaptor(StorageLayer storage) {
@@ -606,6 +618,7 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) {
                     pool.setAuthSecret(secretValue);
                 }
             }
+            pool.setAuthSecretUUID(spd.getSecretUUID());
 
             /**
              * On large (RBD) storage pools it can take up to a couple of minutes
@@ -1642,6 +1655,13 @@ public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMSt
          */
 
         KVMStoragePool srcPool = disk.getPool();
+        if (ArrayUtils.isNotEmpty(srcPassphrase) && ENCRYPTED_COPY_POOL_TYPES.contains(srcPool.getType())
+                && ENCRYPTED_COPY_POOL_TYPES.contains(destPool.getType())) {
+            // the volume keeps its passphrase when it is copied to another pool
+            return copyEncryptedPhysicalDisk(disk, name, destPool, timeout, srcPassphrase,
+                    ArrayUtils.isNotEmpty(dstPassphrase) ? dstPassphrase : srcPassphrase);
+        }
+
         /* Linstor images are always stored as RAW, but Linstor uses qcow2 in DB,
            to support snapshots(backuped) as qcow2 files. */
         PhysicalDiskFormat sourceFormat = srcPool.getType() != StoragePoolType.Linstor ?
@@ -1796,6 +1816,99 @@ to support snapshots(backuped) as qcow2 files. */
         return newDisk;
     }
 
+    /**
+     * Copies an encrypted volume to another pool. The data is read through the encryption of the source and written
+     * through the encryption of the destination, which is created up front: an RBD image gets a native librbd LUKS2
+     * header, a volume on a file based pool is a qemu LUKS encrypted QCOW2 file. Copying the encrypted data as is
+     * would not do: the destination may need another kind of encryption than the source, and an encrypted RBD clone
+     * reads the data it inherits from its (plaintext) parent unencrypted. Any backing chain of the source (an RBD
+     * parent or a QCOW2 backing file) is flattened into the destination.
+     */
+    private KVMPhysicalDisk copyEncryptedPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout,
+            byte[] srcPassphrase, byte[] dstPassphrase) {
+        KVMStoragePool srcPool = disk.getPool();
+        boolean srcOnRbd = srcPool.getType() == StoragePoolType.RBD;
+        boolean destOnRbd = destPool.getType() == StoragePoolType.RBD;
+        // the size of the disk as seen by the Instance: the LUKS2 header of an RBD image is carved out of the image
+        long guestSize = srcOnRbd ? disk.getVirtualSize() - LUKS2_HEADER_RESERVE_BYTES : disk.getVirtualSize();
+        logger.debug("Copying encrypted volume {} from pool {} to {} in pool {}, with a size of {}", disk.getPath(), srcPool.getUuid(),
+                name, destPool.getUuid(), toHumanReadableSize(guestSize));
+
+        KVMPhysicalDisk newDisk = destPool.createPhysicalDisk(name, destOnRbd ? PhysicalDiskFormat.RAW : PhysicalDiskFormat.QCOW2,
+                Storage.ProvisioningType.THIN, destOnRbd ? guestSize + LUKS2_HEADER_RESERVE_BYTES : guestSize, dstPassphrase);
+
+        List<Path> cephAuthFiles = new ArrayList<>();
+        try (KeyFile srcKeyFile = new KeyFile(srcPassphrase); KeyFile destKeyFile = new KeyFile(dstPassphrase)) {
+            QemuImageOptions srcImageOpts = encryptedImageOptions(srcPool, disk.getName(), disk.getPath(),
+                    disk.getFormat() == PhysicalDiskFormat.QCOW2 ? PhysicalDiskFormat.QCOW2 : PhysicalDiskFormat.LUKS, SRC_LUKS_SECRET_ID, cephAuthFiles);
+            srcImageOpts.setImageOptsFlag(true);
+            QemuImageOptions destImageOpts = encryptedImageOptions(destPool, name, newDisk.getPath(), PhysicalDiskFormat.QCOW2,
+                    DEST_LUKS_SECRET_ID, cephAuthFiles);
+
+            QemuImgFile srcFile = new QemuImgFile(disk.getPath(), srcOnRbd ? PhysicalDiskFormat.RAW : disk.getFormat());
+            List<QemuObject> secrets = List.of(createSecretObject(SRC_LUKS_SECRET_ID, srcKeyFile), createSecretObject(DEST_LUKS_SECRET_ID, destKeyFile));
+            new QemuImg(timeout).convert(srcFile, null, null, null, secrets, srcImageOpts, destImageOpts, null, false, false, false, false, null, null);
+        } catch (IOException | QemuImgException | LibvirtException e) {
+            logger.error("Failed to copy encrypted volume {} to {} in pool {}: {}", disk.getPath(), name, destPool.getUuid(), e.getMessage());
+            try {
+                destPool.deletePhysicalDisk(name, destOnRbd ? Storage.ImageFormat.RAW : Storage.ImageFormat.QCOW2);
+            } catch (Exception ex) {
+                logger.warn("Failed to remove {} from pool {} after the failed copy: {}", name, destPool.getUuid(), ex.getMessage());
+            }
+            throw new CloudRuntimeException(String.format("Failed to copy encrypted volume %s to %s", disk.getPath(), name), e);
+        } finally {
+            for (Path file : cephAuthFiles) {
+                try {
+                    Files.deleteIfExists(file);
+                } catch (IOException e) {
+                    logger.warn("Failed to remove temporary file {}: {}", file, e.getMessage());
+                }
+            }
+        }
+
+        newDisk.setFormat(destOnRbd ? PhysicalDiskFormat.RAW : PhysicalDiskFormat.QCOW2);
+        newDisk.setVirtualSize(guestSize);
+        newDisk.setQemuEncryptFormat(destOnRbd ? QemuObject.EncryptFormat.LUKS2 : QemuObject.EncryptFormat.LUKS);
+        return newDisk;
+    }
+
+    /**
+     * qemu image options addressing an encrypted volume: an RBD image (with librbd LUKS2 encryption, authenticated through
+     * a temporary ceph.conf and keyring that are added to {@code cephAuthFiles}) or a file of the given format.
+     */
+    private QemuImageOptions encryptedImageOptions(KVMStoragePool pool, String rbdImage, String filePath, PhysicalDiskFormat fileFormat,
+            String secretId, List<Path> cephAuthFiles) throws IOException {
+        if (pool.getType() != StoragePoolType.RBD) {
+            return new QemuImageOptions(fileFormat, filePath, secretId);
+        }
+        Path keyring = Files.createTempFile("cs-ceph-", ".keyring", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------")));
+        cephAuthFiles.add(keyring);
+        Files.writeString(keyring, "[client." + pool.getAuthUserName() + "]\n\tkey = " + pool.getAuthSecret() + "\n");
+        Path conf = Files.createTempFile("cs-ceph-", ".conf", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------")));
+        cephAuthFiles.add(conf);
+        String monHost = RbdUtils.getRbdMonitors(pool.getSourceHost(), pool.getSourcePort());
+        Files.writeString(conf, "[global]\nmon_host = " + monHost + "\nkeyring = " + keyring + "\n");
+
+        Map<String, String> options = new HashMap<>();
+        options.put("driver", "rbd");
+        options.put("pool", pool.getSourceDir());
+        options.put("image", rbdImage);
+        options.put("conf", conf.toString());
+        if (pool.getAuthUserName() != null) {
+            options.put("user", pool.getAuthUserName());
+        }
+        options.put("encrypt.format", CryptSetup.LuksType.LUKS2.toString());
+        options.put("encrypt.key-secret", secretId);
+        return new QemuImageOptions(options);
+    }
+
+    private QemuObject createSecretObject(String secretId, KeyFile keyFile) {
+        Map<QemuObject.ObjectParameter, String> params = new EnumMap<>(QemuObject.ObjectParameter.class);
+        params.put(QemuObject.ObjectParameter.ID, secretId);
+        params.put(QemuObject.ObjectParameter.FILE, keyFile.toString());
+        return new QemuObject(QemuObject.ObjectType.SECRET, params);
+    }
+
     @Override
     public boolean refresh(KVMStoragePool pool) {
         LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java

@@ -56,6 +56,7 @@ public class LibvirtStoragePool implements KVMStoragePool {
     protected StoragePool _pool;
     protected String authUsername;
     protected String authSecret;
+    protected String authSecretUUID;
     protected String sourceHost;
     protected int sourcePort;
     protected String sourceDir;
@@ -309,6 +310,15 @@ public void setType(StoragePoolType type) {
         this.type = type;
     }
 
+    @Override
+    public String getAuthSecretUUID() {
+        return authSecretUUID;
+    }
+
+    public void setAuthSecretUUID(String authSecretUUID) {
+        this.authSecretUUID = authSecretUUID;
+    }
+
     public StoragePool getPool() {
         return this._pool;
     }