From f32e2d5dda3a88bc933ab7c44db232707751c227 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Thu, 28 Mar 2024 17:10:11 +0530 Subject: [PATCH 1/2] rbd: add additional space for encrypted volumes issue: when a block-mode pvc is created with encryption enabled there is some space reserved for the encryption metadata. Which doesn't allows users to write extact amount of data that they have requested for. solution: create pvc with extra space needed for the encryption metadata. The extra space is added during the CreateVolume and ExpandVolume operations. And while returning the response remove the extra space so the client/user gets the requested size reported. Signed-off-by: Praveen M --- internal/rbd/controllerserver.go | 22 +++++++++++ internal/rbd/encryption.go | 8 ++++ internal/rbd/rbd_util.go | 53 ++++++++++++++++++++++++-- internal/util/cryptsetup/cryptsetup.go | 13 ++++++- 4 files changed, 91 insertions(+), 5 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 8283970f563..1802693f412 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -1226,6 +1226,17 @@ func (cs *ControllerServer) CreateSnapshot( return nil, status.Error(codes.Internal, err.Error()) } + err = vol.Connect(cr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer vol.Destroy(ctx) + + err = vol.getImageInfo() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + csiSnap, err := vol.toSnapshot().ToCSI(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -1285,6 +1296,17 @@ func cloneFromSnapshot( } } + err = rbdSnap.Connect(cr) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer rbdSnap.Destroy(ctx) + + err = rbdSnap.getImageInfo() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + csiSnap, err := rbdSnap.ToCSI(ctx) if err != nil { return nil, status.Error(codes.Internal, err.Error()) diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go index eadef94fbbb..f8e78d73f68 100644 --- a/internal/rbd/encryption.go +++ b/internal/rbd/encryption.go @@ -60,6 +60,9 @@ const ( metadataDEK = "rbd.csi.ceph.com/dek" oldMetadataDEK = ".rbd.csi.ceph.com/dek" + // luks2 header size metadata key. + luks2HeaderSizeKey = "rbd.csi.ceph.com/luks2HeaderSize" + encryptionPassphraseSize = 20 // rbdDefaultEncryptionType is the default to use when the @@ -131,6 +134,11 @@ func (ri *rbdImage) setupBlockEncryption(ctx context.Context) error { return err } + err = ri.SetMetadata(luks2HeaderSizeKey, strconv.FormatUint(cryptsetup.Luks2HeaderSize, 10)) + if err != nil { + return fmt.Errorf("failed to save %s metadata on image: %w", luks2HeaderSizeKey, err) + } + err = ri.ensureEncryptionMetadataSet(rbdImageEncryptionPrepared) if err != nil { log.ErrorLog(ctx, "failed to save encryption status, deleting "+ diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 274981bfe86..c7d31c3ede7 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -30,6 +30,7 @@ import ( "github.com/ceph/ceph-csi/internal/rbd/types" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/cryptsetup" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/go-ceph/rados" @@ -453,8 +454,16 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er return fmt.Errorf("failed to get IOContext: %w", err) } - err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, - uint64(util.RoundOffVolSize(pOpts.VolSize)*helpers.MiB), options) + size := uint64(util.RoundOffVolSize(pOpts.VolSize) * helpers.MiB) + if pOpts.isBlockEncrypted() { + // When a block-mode PVC is created with encryption enabled, + // some space is reserved for the LUKS2 header. + // Add the LUKS2 header size to the image size so that the user has at least + // the requested size. + size += cryptsetup.Luks2HeaderSize + } + + err = librbd.CreateImage(pOpts.ioctx, pOpts.RbdImageName, size, options) if err != nil { return fmt.Errorf("failed to create rbd image: %w", err) } @@ -1643,6 +1652,26 @@ func (ri *rbdImage) GetCreationTime(ctx context.Context) (*time.Time, error) { return ri.CreatedAt, nil } +// getLuks2HeaderSizeSet returns the value of the LUKS2 header size +// set in the image metadata (size returned in MiB). +func (ri *rbdImage) getLuks2HeaderSizeSet() (uint64, error) { + value, err := ri.GetMetadata(luks2HeaderSizeKey) + if err != nil { + if !errors.Is(err, librbd.ErrNotFound) { + return 0, err + } + + return 0, nil + } + + headerSize, parseErr := strconv.ParseUint(value, 10, 64) + if parseErr != nil { + return 0, parseErr + } + + return headerSize, nil +} + // getImageInfo queries rbd about the given image and returns its metadata, and returns // ErrImageNotFound if provided image is not found. func (ri *rbdImage) getImageInfo() error { @@ -1659,6 +1688,14 @@ func (ri *rbdImage) getImageInfo() error { // TODO: can rv.VolSize not be a uint64? Or initialize it to -1? ri.VolSize = int64(imageInfo.Size) + // If the luks2HeaderSizeKey metadata is set + // reduce the extra size of the LUKS header from the image size. + headerSize, err := ri.getLuks2HeaderSizeSet() + if err != nil { + return err + } + ri.VolSize -= int64(headerSize) + features, err := image.GetFeatures() if err != nil { return err @@ -1908,7 +1945,17 @@ func (ri *rbdImage) resize(newSize int64) error { } defer image.Close() - err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB)) + size := uint64(util.RoundOffVolSize(newSize) * helpers.MiB) + + // If the luks2HeaderSizeKey metadata is set + // add the extra size of the LUKS header to the image size. + headerSize, err := ri.getLuks2HeaderSizeSet() + if err != nil { + return err + } + size += headerSize + + err = image.Resize(size) if err != nil { return err } diff --git a/internal/util/cryptsetup/cryptsetup.go b/internal/util/cryptsetup/cryptsetup.go index 7423fb417a8..9f58f1b55f9 100644 --- a/internal/util/cryptsetup/cryptsetup.go +++ b/internal/util/cryptsetup/cryptsetup.go @@ -30,6 +30,8 @@ import ( "github.com/ceph/ceph-csi/internal/util/file" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/ceph-csi/internal/util/stripsecrets" + + "k8s.io/cloud-provider/volume/helpers" ) const ( @@ -37,7 +39,10 @@ const ( ExecutionTimeout = 2*time.Minute + 30*time.Second // Limit memory used by Argon2i PBKDF to 32 MiB. - pkdbfMemoryLimit = 32 << 10 // 32768 KiB + cryptsetupPBKDFMemoryLimit = 32 << 10 // 32768 KiB + luks2MetadataSize = 32 << 7 // 4096 KiB + luks2KeySlotsSize = 32 << 8 // 8192 KiB + Luks2HeaderSize = uint64((((2 * luks2MetadataSize) + luks2KeySlotsSize) * helpers.KiB)) ) // LuksWrapper is a struct that provides a context-aware wrapper around cryptsetup commands. @@ -74,8 +79,12 @@ func (l *luksWrapper) Format(devicePath, passphrase string) (string, string, err "luks2", "--hash", "sha256", + "--luks2-metadata-size", + strconv.Itoa(luks2MetadataSize)+"k", + "--luks2-keyslots-size", + strconv.Itoa(luks2KeySlotsSize)+"k", "--pbkdf-memory", - strconv.Itoa(pkdbfMemoryLimit), + strconv.Itoa(cryptsetupPBKDFMemoryLimit), devicePath, "-d", "/dev/stdin") From a37be5106094178856ea937898122c07a98d384d Mon Sep 17 00:00:00 2001 From: Praveen M Date: Mon, 29 Apr 2024 12:07:43 +0530 Subject: [PATCH 2/2] e2e: test to verify the encrypted image size This commit adds testcase for the create/resize/clone/restore operations for a RBD block PVC and validate imageSize and deviceSize. Signed-off-by: Praveen M --- PendingReleaseNotes.md | 1 + e2e/rbd.go | 211 +++++++++++++++++++++++++++++++++++++++++ e2e/rbd_helper.go | 26 +++++ 3 files changed, 238 insertions(+) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 76de0dfb873..16f713a16ea 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -11,5 +11,6 @@ value [PR](https://github.com/ceph/ceph-csi/pull/4887) - cephfs: support omap data store in radosnamespace [PR](https://github.com/ceph/ceph-csi/pull/4661) - helm: Support setting nodepluigin and provisioner annotations +- rbd: add additional space for encrypted volumes for Luks2 header in [PR](https://github.com/ceph/ceph-csi/pull/4582) ## NOTE diff --git a/e2e/rbd.go b/e2e/rbd.go index fddba2937d5..90a6412acb1 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -24,13 +24,16 @@ import ( "time" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/cryptsetup" . "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" + "k8s.io/cloud-provider/volume/helpers" "k8s.io/kubernetes/test/e2e/framework" e2edebug "k8s.io/kubernetes/test/e2e/framework/debug" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" @@ -1998,6 +2001,214 @@ var _ = Describe("RBD", func() { } }) + By("create/resize/clone/restore a encrypted block pvc and verify the image size", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{"encrypted": "true", "encryptionType": util.EncryptionTypeBlock.String()}, + deletePolicy) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + + err = createRBDSnapshotClass(f) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + defer func() { + err = deleteRBDSnapshotClass() + if err != nil { + framework.Failf("failed to delete VolumeSnapshotClass: %v", err) + } + }() + + var ( + imageSize uint64 + resizeImageSize uint64 + sizeInBytes int64 + ) + + //nolint:goconst // The string "1Gi" is used multiple times in rbd.go, so it's not a const value. + pvcSize := "1Gi" + if sizeInBytes, err = helpers.RoundUpToB(resource.MustParse(pvcSize)); err != nil { + framework.Failf("failed to parse pvc size: %v", err) + } + imageSize = uint64(sizeInBytes) + cryptsetup.Luks2HeaderSize + + pvc, err := loadPVC(rawPvcPath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + pvc.Namespace = f.UniqueName + pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + + app, err := loadApp(rawAppPath) + if err != nil { + framework.Failf("failed to load application: %v", err) + } + labelKey := "app" + labelValue := "rbd-pod-block-encrypted" + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValue), + } + + app.Labels = map[string]string{labelKey: labelValue} + app.Namespace = f.UniqueName + err = createPVCAndApp("", f, pvc, app, deployTimeout) + if err != nil { + framework.Failf("failed to create PVC and application: %v", err) + } + + // validate created backend rbd images + err = validateImageSize(f, pvc, imageSize) + if err != nil { + framework.Failf("failed to validate image size: %v", err) + } + err = checkDeviceSize(app, f, &opt, pvcSize) + if err != nil { + framework.Failf("failed to validate device size: %v", err) + } + + // create clone PVC and validate the image size + labelValueClonePod := "rbd-pod-block-encrypted-clone" + optClonePod := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValueClonePod), + } + + pvcClone, err := loadPVC(pvcBlockSmartClonePath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + pvcClone.Namespace = f.UniqueName + + appClone, err := loadApp(appBlockSmartClonePath) + if err != nil { + framework.Failf("failed to load application: %v", err) + } + appClone.Namespace = f.UniqueName + appClone.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name + appClone.Labels = map[string]string{labelKey: labelValueClonePod} + + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + framework.Failf("failed to create clone PVC and application : %v", err) + } + + err = validateImageSize(f, pvcClone, imageSize) + if err != nil { + framework.Failf("failed to validate image size: %v", err) + } + err = checkDeviceSize(appClone, f, &optClonePod, pvcSize) + if err != nil { + framework.Failf("failed to validate device size: %v", err) + } + + // create snapshot and restore PVC and validate the image size + labelValueRestorePod := "rbd-pod-block-encrypted-restore" + optRestorePod := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelKey, labelValueRestorePod), + } + snap := getSnapshot(snapshotPath) + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + + err = createSnapshot(&snap, deployTimeout) + if err != nil { + framework.Failf("failed to create snapshot: %v", err) + } + + pvcRestore, err := loadPVC(pvcBlockRestorePath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + pvcRestore.Spec.DataSource.Name = snap.Name + pvcRestore.Spec.VolumeMode = pvc.Spec.VolumeMode + pvcRestore.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + pvcRestore.Namespace = f.UniqueName + + appRestore, err := loadApp(appBlockRestorePath) + if err != nil { + framework.Failf("failed to load application: %v", err) + } + appRestore.Namespace = f.UniqueName + appRestore.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcRestore.Name + appRestore.Labels = map[string]string{labelKey: labelValueRestorePod} + + err = createPVCAndApp("", f, pvcRestore, appRestore, deployTimeout) + if err != nil { + framework.Failf("failed to create clone PVC and application : %v", err) + } + + err = validateImageSize(f, pvcRestore, imageSize) + if err != nil { + framework.Failf("failed to validate image size: %v", err) + } + err = checkDeviceSize(appRestore, f, &optRestorePod, pvcSize) + if err != nil { + framework.Failf("failed to validate device size: %v", err) + } + + // resize PVC and validate the image size + resizePVCSize := "2Gi" + if sizeInBytes, err = helpers.RoundUpToB(resource.MustParse(resizePVCSize)); err != nil { + framework.Failf("failed to parse resize pvc size: %v", err) + } + resizeImageSize = uint64(sizeInBytes) + cryptsetup.Luks2HeaderSize + + err = expandPVCSize(f.ClientSet, pvc, resizePVCSize, deployTimeout) + if err != nil { + framework.Failf("failed to expand pvc size: %v", err) + } + // wait for application pod to come up after resize + err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout, noError) + if err != nil { + framework.Failf("timeout waiting for pod to be in running state: %v", err) + } + + err = validateImageSize(f, pvc, resizeImageSize) + if err != nil { + framework.Failf("failed to validate image size after resize: %v", err) + } + err = checkDeviceSize(app, f, &opt, resizePVCSize) + if err != nil { + framework.Failf("failed to validate device size after resize: %v", err) + } + + // delete resources + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + framework.Failf("failed to delete pvc and app: %v", err) + } + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + framework.Failf("failed to delete clone pvc and app: %v", err) + } + err = deletePVCAndApp("", f, pvcRestore, appRestore) + if err != nil { + framework.Failf("failed to delete clone pvc and app: %v", err) + } + err = deleteSnapshot(&snap, deployTimeout) + if err != nil { + framework.Failf("failed to delete snapshot: %v", err) + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType) + }) + ByFileAndBlockEncryption("create a PVC and bind it to an app using rbd-nbd mounter with encryption", func( validator encryptionValidateFunc, _ validateFunc, encType util.EncryptionType, ) { diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index c2ef08b67d4..deddb2ec2a8 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -1089,6 +1089,7 @@ type imageInfo struct { StripeUnit int `json:"stripe_unit"` StripeCount int `json:"stripe_count"` ObjectSize int `json:"object_size"` + Size uint64 `json:"size"` } // getImageInfo queries rbd about the given image and returns its metadata, and returns @@ -1166,3 +1167,28 @@ func validateStripe(f *framework.Framework, return nil } + +// validateImageSize validates the size of the image. +func validateImageSize(f *framework.Framework, pvc *v1.PersistentVolumeClaim, imageSize uint64) error { + var imgInfo imageInfo + imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f) + if err != nil { + return err + } + + imgInfoStr, err := getImageInfo(f, imageData.imageName, defaultRBDPool) + if err != nil { + return err + } + + err = json.Unmarshal([]byte(imgInfoStr), &imgInfo) + if err != nil { + return fmt.Errorf("unmarshal failed: %w. raw buffer response: %s", err, imgInfoStr) + } + + if imgInfo.Size != imageSize { + return fmt.Errorf("image %s size %d does not match expected %d", imgInfo.Name, imgInfo.Size, imageSize) + } + + return nil +}