Merge pull request #34 from redpanda-data/ts/add-secrets-module

Add secrets module

README.md

@@ -22,6 +22,10 @@ refer to the [./api](./api) directory.
 The `net` module is a small utility module with helper functions for working with
 URL addresses.
 
+## Secrets Module
+
+The `secrets` module contains code to access cloud provider secrets.
+
 ## Radpanda Admin Client Module
 
 The `rpadmin` module is Redpanda [Admin API](https://docs.redpanda.com/api/admin-api/) client.

secrets/aws.go

@@ -0,0 +1,77 @@
+package secrets
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/url"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
+)
+
+type awsSecretsManager struct {
+	client *secretsmanager.Client
+	logger *slog.Logger
+}
+
+func newAWSSecretsManager(ctx context.Context, logger *slog.Logger, url *url.URL) (secretAPI, error) {
+	cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(getRegion(url.Host)))
+	if err != nil {
+		return nil, fmt.Errorf("failed to load AWS config: %w", err)
+	}
+
+	return &awsSecretsManager{
+		client: secretsmanager.NewFromConfig(cfg),
+		logger: logger,
+	}, nil
+}
+
+func (a *awsSecretsManager) getSecretValue(ctx context.Context, key string) (string, bool) {
+	value, err := a.client.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{
+		SecretId: &key,
+	})
+	if err != nil {
+		var nf *types.ResourceNotFoundException
+		if !errors.As(err, &nf) {
+			a.logger.With("error", err, "key", key).Error("Failed to look up secret")
+		}
+		return "", false
+	}
+
+	return *value.SecretString, true
+}
+
+func (a *awsSecretsManager) checkSecretExists(ctx context.Context, key string) bool {
+	secrets, err := a.client.ListSecrets(ctx, &secretsmanager.ListSecretsInput{
+		Filters: []types.Filter{
+			{
+				// this is a prefix check
+				Key:    types.FilterNameStringTypeName,
+				Values: []string{key},
+			},
+		},
+	})
+	if err != nil {
+		return false
+	}
+
+	// we need to make sure a secret with this specific key exists
+	for _, secret := range secrets.SecretList {
+		if *secret.Name == key {
+			return true
+		}
+	}
+
+	return false
+}
+
+func getRegion(host string) string {
+	endpoint := strings.TrimPrefix(host, "secretsmanager.")
+	region := strings.TrimSuffix(endpoint, ".amazonaws.com")
+
+	return region
+}

secrets/aws_test.go

@@ -0,0 +1,45 @@
+package secrets
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_getRegion(t *testing.T) {
+	type args struct {
+		host string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "should accept region as endpoint",
+			args: args{
+				host: "eu-west-1",
+			},
+			want: "eu-west-1",
+		},
+		{
+			name: "should get region from regional AWS endpoint",
+			args: args{
+				host: "eu-west-1.amazonaws.com",
+			},
+			want: "eu-west-1",
+		},
+		{
+			name: "should get region from Secrets Manager endpoint",
+			args: args{
+				host: "secretsmanager.eu-west-1.amazonaws.com",
+			},
+			want: "eu-west-1",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, getRegion(tt.args.host), "getRegion(%v)", tt.args.host)
+		})
+	}
+}

secrets/az.go

@@ -0,0 +1,59 @@
+package secrets
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/url"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+const latestVersion = ""
+
+type azSecretsManager struct {
+	client *azsecrets.Client
+	logger *slog.Logger
+}
+
+func newAzSecretsManager(_ context.Context, logger *slog.Logger, url *url.URL) (secretAPI, error) {
+	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to obtain Azure credentials: %w", err)
+	}
+
+	client, err := azsecrets.NewClient("https://"+url.Host, cred, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create secretmanager client: %w", err)
+	}
+
+	return &azSecretsManager{
+		client: client,
+		logger: logger,
+	}, nil
+}
+
+func (a *azSecretsManager) getSecretValue(ctx context.Context, key string) (string, bool) {
+	resp, err := a.client.GetSecret(ctx, key, latestVersion, nil)
+	if err != nil {
+		if status.Code(err) != codes.NotFound {
+			a.logger.With("error", err, "key", key).Error("Failed to look up secret")
+		}
+		return "", false
+	}
+
+	return *resp.Value, true
+}
+
+func (a *azSecretsManager) checkSecretExists(ctx context.Context, key string) bool {
+	pager := a.client.NewListSecretVersionsPager(key, nil)
+	if !pager.More() {
+		return false
+	}
+
+	page, err := pager.NextPage(ctx)
+	return err == nil && len(page.Value) > 0
+}

secrets/gcp.go

@@ -0,0 +1,62 @@
+package secrets
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/url"
+
+	secretmanager "cloud.google.com/go/secretmanager/apiv1"
+	"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type gcpSecretsManager struct {
+	client    *secretmanager.Client
+	projectID string
+	logger    *slog.Logger
+}
+
+func newGCPSecretsManager(ctx context.Context, logger *slog.Logger, url *url.URL) (secretAPI, error) {
+	client, err := secretmanager.NewClient(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create secretmanager client: %w", err)
+	}
+
+	return &gcpSecretsManager{
+		client:    client,
+		projectID: url.Host,
+		logger:    logger,
+	}, nil
+}
+
+func (g *gcpSecretsManager) getSecretValue(ctx context.Context, key string) (string, bool) {
+	resp, err := g.client.AccessSecretVersion(ctx, &secretmanagerpb.AccessSecretVersionRequest{
+		Name: g.getLatestSecretID(key),
+	})
+	if err != nil {
+		if status.Code(err) != codes.NotFound {
+			g.logger.With("error", err, "key", key).Error("Failed to look up secret")
+		}
+		return "", false
+	}
+
+	value := string(resp.Payload.Data)
+	return value, true
+}
+
+func (g *gcpSecretsManager) checkSecretExists(ctx context.Context, key string) bool {
+	_, err := g.client.GetSecret(ctx, &secretmanagerpb.GetSecretRequest{
+		Name: g.getSecretID(key),
+	})
+	return err == nil
+}
+
+func (g *gcpSecretsManager) getLatestSecretID(key string) string {
+	return fmt.Sprintf("%v/versions/latest", g.getSecretID(key))
+}
+
+func (g *gcpSecretsManager) getSecretID(key string) string {
+	return fmt.Sprintf("projects/%v/secrets/%v", g.projectID, key)
+}

secrets/generic.go

@@ -0,0 +1,99 @@
+// Copyright 2024 Redpanda Data, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package secrets
+
+import (
+	"context"
+	"log/slog"
+	"net/url"
+	"strings"
+
+	"github.com/tidwall/gjson"
+)
+
+// prefix used to reference secrets from external secret managers, to differentiate them from environment variables
+const secretPrefix = "secrets."
+
+type secretAPI interface {
+	getSecretValue(context.Context, string) (string, bool)
+	checkSecretExists(context.Context, string) bool
+}
+
+type createSecretsManagerFn func(ctx context.Context, logger *slog.Logger, url *url.URL) (secretAPI, error)
+
+type secretManager struct {
+	secretAPI secretAPI
+	prefix    string
+}
+
+func (s *secretManager) lookup(ctx context.Context, key string) (string, bool) {
+	secretName, field, ok := s.trimPrefixAndSplit(key)
+	if !ok {
+		return "", false
+	}
+
+	value, found := s.secretAPI.getSecretValue(ctx, secretName)
+	if !found {
+		return "", false
+	}
+
+	if field == "" {
+		return value, true
+	}
+
+	return getJSONValue(value, field)
+}
+
+func (s *secretManager) exists(ctx context.Context, key string) bool {
+	secretName, _, ok := s.trimPrefixAndSplit(key)
+	if !ok {
+		return false
+	}
+
+	return s.secretAPI.checkSecretExists(ctx, secretName)
+}
+
+func newSecretManager(ctx context.Context, logger *slog.Logger, url *url.URL, createSecretsManagerFn createSecretsManagerFn) (LookupFn, ExistsFn, error) {
+	secretsManager, err := createSecretsManagerFn(ctx, logger, url)
+	if err != nil {
+		return nil, nil, err
+	}
+	secretManager := &secretManager{
+		secretAPI: secretsManager,
+		prefix:    strings.TrimPrefix(url.Path, "/"),
+	}
+
+	return secretManager.lookup, secretManager.exists, nil
+}
+
+// trims the secret prefix and returns full secret ID with JSON field reference
+func (s *secretManager) trimPrefixAndSplit(key string) (string, string, bool) {
+	if !strings.HasPrefix(key, secretPrefix) {
+		return "", "", false
+	}
+
+	key = strings.TrimPrefix(key, secretPrefix)
+	if strings.Contains(key, ".") {
+		parts := strings.SplitN(key, ".", 2)
+		return s.prefix + parts[0], parts[1], true
+	}
+
+	return s.prefix + key, "", true
+}
+
+func getJSONValue(json string, field string) (string, bool) {
+	result := gjson.Get(json, field)
+	return result.String(), result.Exists()
+}