linstor: fix encrypted volume snapshot backup and restore

Encrypted Linstor volumes use a LUKS layer inside the DRBD stack, so
the storage-layer snapshot device holds ciphertext while the DRBD
device CloudStack restores to is the decrypted view. Backing up the
raw snapshot and writing it back to the decrypted device corrupted the
volume (different data, unbootable root).

Back up encrypted snapshots from the decrypted DRBD device (forcing
the temporary-resource path) and store them as a LUKS-encrypted qcow2
using the volume passphrase, so snapshots are not kept in clear text
on secondary storage. On revert, decrypt the qcow2 and write plaintext
to the DRBD device; the LUKS layer re-encrypts it. The qemu-img shrink
is skipped for encrypted volumes (the DRBD device is already
net-sized).

Add an integration test (test_linstor_encrypted_snapshots.py): the
encrypted-root snapshot revert round-trip, that
create-volume-from-encrypted-snapshot is rejected by CloudStack core,
and a best-effort check that the backed-up qcow2 is LUKS-encrypted at
rest.

Author: rp-

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java

@@ -18,6 +18,10 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.NfsTO;
@@ -31,9 +35,11 @@
 import com.cloud.utils.script.Script;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
+import org.apache.cloudstack.utils.cryptsetup.KeyFile;
 import org.apache.cloudstack.utils.qemu.QemuImg;
 import org.apache.cloudstack.utils.qemu.QemuImgException;
 import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.cloudstack.utils.qemu.QemuObject;
 import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -83,6 +89,7 @@ private String convertImageToQCow2(
         final String srcPath,
         final SnapshotObjectTO dst,
         final KVMStoragePool secondaryPool,
+        final byte[] passphrase,
         int waitMilliSeconds
         )
         throws LibvirtException, QemuImgException, IOException
@@ -94,9 +101,22 @@ private String convertImageToQCow2(
         final QemuImgFile srcFile = new QemuImgFile(srcPath, QemuImg.PhysicalDiskFormat.RAW);
         final QemuImgFile dstFile = new QemuImgFile(dstPath, QemuImg.PhysicalDiskFormat.QCOW2);
 
-        // NOTE: the qemu img will also contain the drbd metadata at the end
         final QemuImg qemu = new QemuImg(waitMilliSeconds);
-        qemu.convert(srcFile, dstFile);
+        if (passphrase != null && passphrase.length > 0) {
+            // Encrypted volumes are backed up from their decrypted DRBD device, so the snapshot
+            // data here is plaintext. Encrypt the destination qcow2 with the volume's passphrase
+            // (LUKS), so the snapshot is not stored in clear text on secondary storage.
+            try (KeyFile keyFile = new KeyFile(passphrase)) {
+                final Map<String, String> options = new HashMap<>();
+                final List<QemuObject> qemuObjects = new ArrayList<>();
+                qemuObjects.add(QemuObject.prepareSecretForQemuImg(QemuImg.PhysicalDiskFormat.QCOW2,
+                    QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options));
+                qemu.convert(srcFile, dstFile, options, qemuObjects, null, true);
+            }
+        } else {
+            // NOTE: the qemu img will also contain the drbd metadata at the end
+            qemu.convert(srcFile, dstFile);
+        }
         LOGGER.info("Backup snapshot '{}' to '{}'", srcPath, dstPath);
         return dstPath;
     }
@@ -153,14 +173,21 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR
 
             secondaryPool = storagePoolMgr.getStoragePoolByURI(dstDataStore.getUrl());
 
-            String dstPath = convertImageToQCow2(srcPath, dst, secondaryPool, cmd.getWaitInMillSeconds());
+            final byte[] passphrase = src.getVolume() != null ? src.getVolume().getPassphrase() : null;
+            final boolean encrypted = passphrase != null && passphrase.length > 0;
 
-            // resize to real volume size, cutting of drbd metadata
-            String result = qemuShrink(dstPath, src.getVolume().getSize(), cmd.getWaitInMillSeconds());
-            if (result != null) {
-                return new CopyCmdAnswer("qemu-img shrink failed: " + result);
+            String dstPath = convertImageToQCow2(srcPath, dst, secondaryPool, passphrase, cmd.getWaitInMillSeconds());
+
+            if (!encrypted) {
+                // resize to real volume size, cutting of drbd metadata
+                // For encrypted volumes the source is the decrypted DRBD device (already net-sized,
+                // no drbd metadata to cut); shrinking an encrypted qcow2 would also need the secret.
+                String result = qemuShrink(dstPath, src.getVolume().getSize(), cmd.getWaitInMillSeconds());
+                if (result != null) {
+                    return new CopyCmdAnswer("qemu-img shrink failed: " + result);
+                }
+                LOGGER.info("Backup shrunk " + dstPath + " to actual size " + src.getVolume().getSize());
             }
-            LOGGER.info("Backup shrunk " + dstPath + " to actual size " + src.getVolume().getSize());
 
             SnapshotObjectTO snapshot = setCorrectSnapshotSize(dst, dstPath);
             LOGGER.info("Actual file size for '{}' is {}", dstPath, snapshot.getPhysicalSize());
@@ -171,6 +198,9 @@ public CopyCmdAnswer execute(LinstorBackupSnapshotCommand cmd, LibvirtComputingR
             LOGGER.error(error);
             return new CopyCmdAnswer(cmd, e);
         } finally {
+            if (src.getVolume() != null) {
+                src.getVolume().clearPassphrase();
+            }
             cleanupSecondaryPool(secondaryPool);
             if (zfsHidden) {
                 zfsSnapdev(true, src.getPath());

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorRevertBackupSnapshotCommandWrapper.java

@@ -17,6 +17,7 @@
 package com.cloud.hypervisor.kvm.resource.wrapper;
 
 import java.io.File;
+import java.util.Collections;
 
 import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.api.storage.LinstorRevertBackupSnapshotCommand;
@@ -31,9 +32,12 @@
 import org.apache.cloudstack.storage.datastore.util.LinstorUtil;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.cloudstack.utils.cryptsetup.KeyFile;
+import org.apache.cloudstack.utils.qemu.QemuImageOptions;
 import org.apache.cloudstack.utils.qemu.QemuImg;
 import org.apache.cloudstack.utils.qemu.QemuImgException;
 import org.apache.cloudstack.utils.qemu.QemuImgFile;
+import org.apache.cloudstack.utils.qemu.QemuObject;
 import org.joda.time.Duration;
 import org.libvirt.LibvirtException;
 
@@ -43,8 +47,9 @@ public final class LinstorRevertBackupSnapshotCommandWrapper
 {
 
     private void convertQCow2ToRAW(
-            KVMStoragePool pool, final String srcPath, final String dstUuid, int waitMilliSeconds)
-        throws LibvirtException, QemuImgException
+            KVMStoragePool pool, final String srcPath, final String dstUuid, final byte[] passphrase,
+            int waitMilliSeconds)
+        throws LibvirtException, QemuImgException, java.io.IOException
     {
         final String dstPath = pool.getPhysicalDisk(dstUuid).getPath();
         final QemuImgFile srcQemuFile = new QemuImgFile(
@@ -60,7 +65,20 @@ private void convertQCow2ToRAW(
         }
         final QemuImg qemu = new QemuImg(waitMilliSeconds, zeroedDevice, true);
         final QemuImgFile dstFile = new QemuImgFile(dstPath, QemuImg.PhysicalDiskFormat.RAW);
-        qemu.convert(srcQemuFile, dstFile);
+        if (passphrase != null && passphrase.length > 0) {
+            // The backed-up qcow2 is LUKS-encrypted with the volume's passphrase. Decrypt it while
+            // writing plaintext to the (decrypted) DRBD device; the Linstor LUKS layer re-encrypts it,
+            // so no qemu encryption must be applied to the destination.
+            try (KeyFile keyFile = new KeyFile(passphrase)) {
+                final QemuObject srcSecret = QemuObject.prepareSecretForQemuImg(
+                    QemuImg.PhysicalDiskFormat.QCOW2, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", null);
+                final QemuImageOptions srcImageOpts = new QemuImageOptions(
+                    QemuImg.PhysicalDiskFormat.QCOW2, srcPath, "sec0");
+                qemu.convert(srcQemuFile, dstFile, null, Collections.singletonList(srcSecret), srcImageOpts, null, false);
+            }
+        } else {
+            qemu.convert(srcQemuFile, dstFile);
+        }
     }
 
     @Override
@@ -84,10 +102,13 @@ public CopyCmdAnswer execute(LinstorRevertBackupSnapshotCommand cmd, LibvirtComp
             secondaryPool = storagePoolMgr.getStoragePoolByURI(
                 srcDataStore.getUrl() + File.separator + srcFile.getParent());
 
+            // The destination volume is the (same) original volume, whose passphrase the backed-up
+            // qcow2 was encrypted with; use it to decrypt while restoring.
             convertQCow2ToRAW(
                 linstorPool,
                 secondaryPool.getLocalPath() + File.separator + srcFile.getName(),
                 dst.getPath(),
+                dst.getPassphrase(),
                 cmd.getWaitInMillSeconds());
 
             final VolumeObjectTO dstVolume = new VolumeObjectTO();
@@ -99,6 +120,7 @@ public CopyCmdAnswer execute(LinstorRevertBackupSnapshotCommand cmd, LibvirtComp
             logger.error(error);
             return new CopyCmdAnswer(cmd, e);
         } finally {
+            dst.clearPassphrase();
             LinstorBackupSnapshotCommandWrapper.cleanupSecondaryPool(secondaryPool);
         }
     }

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java

@@ -1088,12 +1088,22 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) {
                 VirtualMachineManager.ExecuteInSequence.value());
             cmd.setOptions(options);
 
-            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, pool, rscName);
+            // For encrypted volumes Linstor adds a LUKS layer (DRBD -> LUKS -> STORAGE). The storage
+            // layer snapshot device (getSnapshotPath) therefore only exposes the raw LUKS ciphertext,
+            // while restore writes onto the decrypted DRBD device (/dev/drbd/by-res/.../0). Backing up
+            // the ciphertext and writing it back to the decrypted layer corrupts the volume (and the
+            // shrink to the net volume size would even truncate the ciphertext). So for encrypted
+            // volumes we never read the storage snapshot directly: restore the snapshot into a temporary
+            // resource and back up its decrypted DRBD device instead, symmetric to the restore path.
+            final boolean encrypted = snapshotObject.getBaseVolume().getPassphraseId() != null;
+            Optional<RemoteHostEndPoint> optEP = encrypted ?
+                Optional.empty() : getDiskfullEP(api, pool, rscName);
             Answer answer;
             if (optEP.isPresent()) {
                 answer = optEP.get().sendMessage(cmd);
             } else {
-                logger.debug("No diskfull endpoint found to copy image, creating diskless endpoint");
+                logger.debug("No diskfull endpoint used to copy image (encrypted={}), using temporary resource",
+                    encrypted);
                 answer = copyFromTemporaryResource(api, pool, rscName, snapshotName, snapshotObject, cmd);
             }
             return answer;

test/integration/plugins/linstor/README.md

@@ -48,3 +48,21 @@ nosetests --with-marvin --marvin-config=<marvin-cfg-file> <cloudstack-dir>/test/
 ```
 
 You can also run these tests out of the box with PyDev or PyCharm or whatever.
+
+## Encrypted snapshot tests
+
+`test_linstor_encrypted_snapshots.py` covers the encrypted-volume snapshot round trip
+(create encrypted root disk -> snapshot -> revert / create-volume-from-snapshot) and that the
+backed-up qcow2 on secondary storage is itself LUKS encrypted.
+
+Extra prerequisites:
+
+* At least one KVM host with volume-encryption support (`host.encryptionsupported == true`, i.e.
+  cryptsetup/qemu LUKS available). Tests self-skip if none is found.
+* The Linstor resource group used (`acs-basic`) must be able to add a LUKS layer to its volumes.
+* `lin.backup.snapshots` must be enabled (default) so snapshots are backed up to secondary storage;
+  the test sets it. With it disabled the qcow2 path is not exercised.
+
+```
+nosetests --with-marvin --marvin-config=<marvin-cfg-file> <cloudstack-dir>/test/integration/plugins/linstor/test_linstor_encrypted_snapshots.py --zone=<zone> --hypervisor=kvm
+```