purge data when removing ceph user

go.mod

@@ -3,6 +3,7 @@ module github.com/snapp-incubator/s3-operator
 go 1.20
 
 require (
+	github.com/aws/aws-sdk-go v1.44.275
 	github.com/ceph/go-ceph v0.22.0
 	github.com/go-logr/logr v1.2.3
 	github.com/knadh/koanf/parsers/yaml v0.1.0
@@ -13,7 +14,6 @@ require (
 	github.com/onsi/gomega v1.27.4
 	github.com/opdev/subreconciler v0.0.0-20230302151718-c4c8b5ec17c5
 	github.com/openshift/api v0.0.0-20230503113241-06ec0523a98b
-	github.com/stretchr/testify v1.8.4
 	k8s.io/api v0.26.1
 	k8s.io/apimachinery v0.26.1
 	k8s.io/apiserver v0.26.0
@@ -23,7 +23,6 @@ require (
 )
 
 require (
-	github.com/aws/aws-sdk-go v1.44.275 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
@@ -66,13 +65,11 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.5.0 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.5 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.5 // indirect

go.sum

@@ -340,7 +340,6 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -351,7 +350,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

internal/controllers/s3userclaim/cleaner.go

@@ -8,6 +8,7 @@ import (
 	"github.com/opdev/subreconciler"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -34,7 +35,7 @@ func (r *Reconciler) Cleanup(ctx context.Context) (ctrl.Result, error) {
 }
 
 func (r *Reconciler) removeCephUser(ctx context.Context) (*ctrl.Result, error) {
-	switch err := r.rgwClient.RemoveUser(ctx, admin.User{ID: r.cephUserFullId}); {
+	switch err := r.rgwClient.RemoveUser(ctx, admin.User{ID: r.cephUserFullId, PurgeData: pointer.Int(1)}); {
 	case goerrors.Is(err, admin.ErrNoSuchUser):
 		return subreconciler.ContinueReconciling()
 	case err != nil:

internal/controllers/s3userclaim/s3userclaim_test.go

@@ -20,8 +20,13 @@ import (
 	"context"
 	goerrors "errors"
 	"fmt"
+	"net/http"
 	"regexp"
+	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/ceph/go-ceph/rgw/admin"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -35,6 +40,9 @@ import (
 	s3v1alpha1 "github.com/snapp-incubator/s3-operator/api/v1alpha1"
 	"github.com/snapp-incubator/s3-operator/internal/config"
 	"github.com/snapp-incubator/s3-operator/pkg/consts"
+
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
 )
 
 var _ = Describe("S3UserClaim Controller", func() {
@@ -270,4 +278,145 @@ var _ = Describe("S3UserClaim Controller", func() {
 			}).Should(Succeed())
 		})
 	})
+
+	Context("When removing a S3UserClaim", func() {
+		BeforeEach(func() {
+			s3UserClaim = getS3UserClaim()
+
+			s3User = &s3v1alpha1.S3User{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: s3UserName,
+				},
+			}
+
+			adminSecret = &v1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      adminSecretName,
+					Namespace: s3UserClaimNamespace,
+				},
+			}
+
+			readonlySecret = &v1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      readonlySecretName,
+					Namespace: s3UserClaimNamespace,
+				},
+			}
+			Expect(k8sClient.Create(ctx, s3UserClaim)).To(Succeed())
+			Eventually(func(g Gomega) {
+				_, err := co.GetUser(ctx, cephUser)
+				g.Expect(err).To(BeNil())
+			}).Should(Succeed())
+		})
+
+		It("Should remove the Ceph user without buckets", func() {
+			By("Expect to delete the S3UserClaim successfully")
+			Eventually(func(g Gomega) {
+				g.Expect(k8sClient.Delete(ctx, s3UserClaim)).To(Succeed())
+			}).Should(Succeed())
+
+			By("Expect the related objects are cleaned up by the controller")
+			Eventually(func(g Gomega) {
+				g.Expect(
+					apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: s3User.Name}, s3User)),
+				).To(BeTrue())
+
+				g.Expect(k8sClient.Delete(ctx, adminSecret)).To(Succeed())
+				g.Expect(k8sClient.Delete(ctx, readonlySecret)).To(Succeed())
+
+				_, err := rgwClient.GetUser(ctx, cephUser)
+				g.Expect(goerrors.Is(err, admin.ErrNoSuchUser)).To(BeTrue())
+			}).Should(Succeed())
+		})
+
+		It("Should remove the Ceph user with buckets", func() {
+			gotCephUser, err := rgwClient.GetUser(ctx, cephUser)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(len(gotCephUser.Keys)).To(Equal(2))
+			var s3Keys admin.UserKeySpec
+			if gotCephUser.Keys[0].User == cephUser.ID {
+				s3Keys = gotCephUser.Keys[0]
+			} else if gotCephUser.Keys[1].User == cephUser.ID {
+				s3Keys = gotCephUser.Keys[1]
+			} else {
+				Fail("failed to find the expected ceph user")
+			}
+
+			s3Agent, err := newS3Agent(s3Keys.AccessKey, s3Keys.SecretKey, cfg.Rgw.Endpoint, true)
+			Expect(err).To(BeNil())
+			Expect(s3Agent.createBucket("test-bucket")).To(Succeed())
+
+			By("Expect to delete the S3UserClaim successfully")
+			Eventually(func(g Gomega) {
+				g.Expect(k8sClient.Delete(ctx, s3UserClaim)).To(Succeed())
+			}).Should(Succeed())
+
+			By("Expect the related objects are cleaned up by the controller")
+			Eventually(func(g Gomega) {
+				g.Expect(
+					apierrors.IsNotFound(k8sClient.Get(ctx, types.NamespacedName{Name: s3User.Name}, s3User)),
+				).To(BeTrue())
+
+				g.Expect(k8sClient.Delete(ctx, adminSecret)).To(Succeed())
+				g.Expect(k8sClient.Delete(ctx, readonlySecret)).To(Succeed())
+
+				_, err := rgwClient.GetUser(ctx, cephUser)
+				g.Expect(goerrors.Is(err, admin.ErrNoSuchUser)).To(BeTrue())
+			}).Should(Succeed())
+		})
+	})
 })
+
+// S3Agent wraps the s3.S3 structure to allow for wrapper methods
+type S3Agent struct {
+	Client *s3.S3
+}
+
+func newS3Agent(accessKey, secretKey, endpoint string, debug bool) (*S3Agent, error) {
+	const cephRegion = "us-east-1"
+
+	logLevel := aws.LogOff
+	if debug {
+		logLevel = aws.LogDebug
+	}
+	client := http.Client{
+		Timeout: time.Second * 15,
+	}
+	sess, err := session.NewSession(
+		aws.NewConfig().
+			WithRegion(cephRegion).
+			WithCredentials(credentials.NewStaticCredentials(accessKey, secretKey, "")).
+			WithEndpoint(endpoint).
+			WithS3ForcePathStyle(true).
+			WithMaxRetries(5).
+			WithDisableSSL(true).
+			WithHTTPClient(&client).
+			WithLogLevel(logLevel),
+	)
+	if err != nil {
+		return nil, err
+	}
+	svc := s3.New(sess)
+	return &S3Agent{
+		Client: svc,
+	}, nil
+}
+
+func (s *S3Agent) createBucket(name string) error {
+	bucketInput := &s3.CreateBucketInput{
+		Bucket: &name,
+	}
+	_, err := s.Client.CreateBucket(bucketInput)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			case s3.ErrCodeBucketAlreadyExists:
+				return nil
+			case s3.ErrCodeBucketAlreadyOwnedByYou:
+				return nil
+			}
+		}
+		return fmt.Errorf("failed to create bucket %q. %w", name, err)
+	}
+	return nil
+}