feat: configure default cas backend size (#1655)

Signed-off-by: Miguel Martinez <miguel@chainloop.dev>
chainloop-dev · Dec 14, 2024 · 82d8a6a · 82d8a6a
1 parent f857add
commit 82d8a6a
Show file tree

Hide file tree

Showing 14 changed files with 326 additions and 176 deletions.
diff --git a/app/controlplane/cmd/wire_gen.go b/app/controlplane/cmd/wire_gen.go
diff --git a/app/controlplane/configs/config.devel.yaml b/app/controlplane/configs/config.devel.yaml
@@ -43,7 +43,7 @@ cas_server:
     addr: 0.0.0.0:9001
   insecure: true
   download_url: http://0.0.0.0:8001/download
-
+  default_entry_max_size: 300MB
 credentials_service:
   vault:
     address: ${VAULT_ADDRESS:http://0.0.0.0:8200}

diff --git a/app/controlplane/configs/samples/config.yaml b/app/controlplane/configs/samples/config.yaml
@@ -29,6 +29,7 @@ cas_server:
     addr: 0.0.0.0:9001
   # Where to redirect the user to download artifacts from the CAS
   download_url: http://0.0.0.0:8001/download
+  default_entry_max_size: 1GB
 
 # nats endpoint where to send events
 nats_server:

diff --git a/app/controlplane/internal/conf/controlplane/config/v1/conf.pb.go b/app/controlplane/internal/conf/controlplane/config/v1/conf.pb.go
diff --git a/app/controlplane/internal/conf/controlplane/config/v1/conf.proto b/app/controlplane/internal/conf/controlplane/config/v1/conf.proto
@@ -58,6 +58,11 @@ message Bootstrap {
     // In the form of [scheme]://[host]/path i.e https://cas.chainloop.dev/download
     // https://github.com/chainloop-dev/chainloop/blob/126f47b6c0803eac844b8e3e1a21d582f00e4dc6/app/artifact-cas/internal/service/download.go#L34
     string download_url = 3;
+
+    // Default max size for each entry in the CAS backend
+    // the format is a number followed by a unit, like 100MB, 1GB, etc
+    // Default is 100MB
+    string default_entry_max_size = 4;
   }
 
   // Configuration for onboarding users in organizations with specific roles

diff --git a/app/controlplane/pkg/biz/casbackend.go b/app/controlplane/pkg/biz/casbackend.go
@@ -23,6 +23,8 @@ import (
 	"io"
 	"time"
 
+	"code.cloudfoundry.org/bytefmt"
+	conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1"
 	backend "github.com/chainloop-dev/chainloop/pkg/blobmanager"
 	"github.com/chainloop-dev/chainloop/pkg/blobmanager/azureblob"
 	"github.com/chainloop-dev/chainloop/pkg/blobmanager/oci"
@@ -36,7 +38,6 @@ import (
 type CASBackendProvider string
 
 const (
-	CASBackendDefaultMaxBytes int64 = 100 * 1024 * 1024 // 100MB
 	// Inline, embedded CAS backend
 	CASBackendInline                CASBackendProvider = "INLINE"
 	CASBackendInlineDefaultMaxBytes int64              = 500 * 1024 // 500KB
@@ -111,18 +112,28 @@ type CASBackendReader interface {
 }
 
 type CASBackendUseCase struct {
-	repo      CASBackendRepo
-	logger    *log.Helper
-	credsRW   credentials.ReaderWriter
-	providers backend.Providers
+	repo            CASBackendRepo
+	logger          *log.Helper
+	credsRW         credentials.ReaderWriter
+	providers       backend.Providers
+	MaxBytesDefault int64
 }
 
-func NewCASBackendUseCase(repo CASBackendRepo, credsRW credentials.ReaderWriter, providers backend.Providers, l log.Logger) *CASBackendUseCase {
+func NewCASBackendUseCase(repo CASBackendRepo, credsRW credentials.ReaderWriter, providers backend.Providers, c *conf.Bootstrap_CASServer, l log.Logger) (*CASBackendUseCase, error) {
 	if l == nil {
 		l = log.NewStdLogger(io.Discard)
 	}
 
-	return &CASBackendUseCase{repo, servicelogger.ScopedHelper(l, "biz/CASBackend"), credsRW, providers}
+	var maxBytesDefault uint64 = 100 * 1024 * 1024 // 100MB
+	if c.GetDefaultEntryMaxSize() != "" {
+		var err error
+		maxBytesDefault, err = bytefmt.ToBytes(c.DefaultEntryMaxSize)
+		if err != nil {
+			return nil, fmt.Errorf("invalid CAS backend default max bytes: %w", err)
+		}
+	}
+
+	return &CASBackendUseCase{repo, servicelogger.ScopedHelper(l, "biz/CASBackend"), credsRW, providers, int64(maxBytesDefault)}, nil
 }
 
 func (uc *CASBackendUseCase) List(ctx context.Context, orgID string) ([]*CASBackend, error) {
@@ -253,7 +264,7 @@ func (uc *CASBackendUseCase) Create(ctx context.Context, orgID, name, location,
 	}
 
 	backend, err := uc.repo.Create(ctx, &CASBackendCreateOpts{
-		MaxBytes: CASBackendDefaultMaxBytes,
+		MaxBytes: uc.MaxBytesDefault,
 		Name:     name,
 		CASBackendOpts: &CASBackendOpts{
 			Location: location, SecretName: secretName, Provider: provider, Default: defaultB,

diff --git a/app/controlplane/pkg/biz/casbackend_test.go b/app/controlplane/pkg/biz/casbackend_test.go
@@ -20,6 +20,7 @@ import (
 	"errors"
 	"testing"
 
+	conf "github.com/chainloop-dev/chainloop/app/controlplane/internal/conf/controlplane/config/v1"
 	"github.com/chainloop-dev/chainloop/app/controlplane/pkg/biz"
 	bizMocks "github.com/chainloop-dev/chainloop/app/controlplane/pkg/biz/mocks"
 	backends "github.com/chainloop-dev/chainloop/pkg/blobmanager"
@@ -171,6 +172,99 @@ func (s *casBackendTestSuite) TestPerformValidation() {
 	})
 }
 
+func (s *casBackendTestSuite) TestNewCASBackendUseCase() {
+	assert := assert.New(s.T())
+	const defaultErrorMsg = "byte quantity must be a positive integer with a unit of measurement like M, MB, MiB, G, GiB, or GB"
+
+	tests := []struct {
+		name        string
+		config      *conf.Bootstrap_CASServer
+		expectError bool
+		errorMsg    string
+		wantSize    int64 // Expected size in bytes after parsing
+	}{
+		{
+			name:        "nil config uses default",
+			config:      nil,
+			expectError: false,
+			wantSize:    100 * 1024 * 1024, // 100MB default
+		},
+		{
+			name: "valid size - megabytes",
+			config: &conf.Bootstrap_CASServer{
+				DefaultEntryMaxSize: "100MB",
+			},
+			expectError: false,
+			wantSize:    100 * 1024 * 1024,
+		},
+		{
+			name: "valid size - gigabytes",
+			config: &conf.Bootstrap_CASServer{
+				DefaultEntryMaxSize: "2GB",
+			},
+			expectError: false,
+			wantSize:    2 * 1024 * 1024 * 1024,
+		},
+		{
+			name: "invalid size format",
+			config: &conf.Bootstrap_CASServer{
+				DefaultEntryMaxSize: "invalid",
+			},
+			expectError: true,
+			errorMsg:    defaultErrorMsg,
+			wantSize:    0,
+		},
+		{
+			name: "negative size",
+			config: &conf.Bootstrap_CASServer{
+				DefaultEntryMaxSize: "-100MB",
+			},
+			expectError: true,
+			errorMsg:    defaultErrorMsg,
+			wantSize:    0,
+		},
+		{
+			name: "zero size",
+			config: &conf.Bootstrap_CASServer{
+				DefaultEntryMaxSize: "0",
+			},
+			expectError: true,
+			errorMsg:    defaultErrorMsg,
+			wantSize:    0,
+		},
+		{
+			name: "missing unit",
+			config: &conf.Bootstrap_CASServer{
+				DefaultEntryMaxSize: "100",
+			},
+			expectError: true,
+			errorMsg:    defaultErrorMsg,
+			wantSize:    0,
+		},
+	}
+
+	for _, tc := range tests {
+		s.Run(tc.name, func() {
+			useCase, err := biz.NewCASBackendUseCase(s.repo, s.credsRW,
+				backends.Providers{
+					"OCI": s.backendProvider,
+				}, tc.config, nil)
+
+			if tc.expectError {
+				assert.Error(err)
+				if tc.errorMsg != "" {
+					assert.Contains(err.Error(), tc.errorMsg)
+				}
+				assert.Nil(useCase)
+			} else {
+				assert.NoError(err)
+				assert.NotNil(useCase)
+				assert.Equal(tc.wantSize, useCase.MaxBytesDefault)
+			}
+		})
+	}
+}
+
 // Run all the tests
 func TestCASBackend(t *testing.T) {
 	suite.Run(t, new(casBackendTestSuite))
@@ -188,9 +282,11 @@ func (s *casBackendTestSuite) SetupTest() {
 	s.repo = bizMocks.NewCASBackendRepo(s.T())
 	s.credsRW = credentialsM.NewReaderWriter(s.T())
 	s.backendProvider = blobM.NewProvider(s.T())
-	s.useCase = biz.NewCASBackendUseCase(s.repo, s.credsRW,
+	var err error
+	s.useCase, err = biz.NewCASBackendUseCase(s.repo, s.credsRW,
 		backends.Providers{
 			"OCI": s.backendProvider,
-		}, nil,
+		}, nil, nil,
 	)
+	s.Require().NoError(err)
 }
diff --git a/app/controlplane/pkg/biz/testhelpers/database.go b/app/controlplane/pkg/biz/testhelpers/database.go
@@ -201,6 +201,12 @@ func NewConfData(db *TestDatabase, t *testing.T) *conf.Data {
 	}
 }
 
+func NewCASBackendConfig() *conf.Bootstrap_CASServer {
+	return &conf.Bootstrap_CASServer{
+		DefaultEntryMaxSize: "100MB",
+	}
+}
+
 func NewPromSpec() []*conf.PrometheusIntegrationSpec {
 	return []*conf.PrometheusIntegrationSpec{}
 }

diff --git a/app/controlplane/pkg/biz/testhelpers/wire.go b/app/controlplane/pkg/biz/testhelpers/wire.go
@@ -57,6 +57,7 @@ func WireTestData(*TestDatabase, *testing.T, log.Logger, credentials.ReaderWrite
 			wire.FieldsOf(new(*conf.Data), "Database"),
 			newNatsConnection,
 			auditor.NewAuditLogPublisher,
+			NewCASBackendConfig,
 		),
 	)
 }

diff --git a/app/controlplane/pkg/biz/testhelpers/wire_gen.go b/app/controlplane/pkg/biz/testhelpers/wire_gen.go
diff --git a/deployment/chainloop/Chart.yaml b/deployment/chainloop/Chart.yaml
@@ -7,7 +7,7 @@ description: Chainloop is an open source software supply chain control plane, a
 
 type: application
 # Bump the patch (not minor, not major) version on each change in the Chart Source code
-version: 1.157.0
+version: 1.157.1
 # Do not update appVersion, this is handled automatically by the release process
 appVersion: v0.139.0
 

diff --git a/deployment/chainloop/README.md b/deployment/chainloop/README.md
@@ -725,6 +725,7 @@ chainloop config save \
 | Name                         | Description                                                                                                                                                                                                                                           | Value             |
 | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- |
 | `cas.replicaCount`           | Number of replicas                                                                                                                                                                                                                                    | `2`               |
+| `cas.defaultMaxEntrySize`    | Maximum size for each entry in the CAS backend, default 100MB                                                                                                                                                                                         |                   |
 | `cas.image.registry`         | Image registry                                                                                                                                                                                                                                        | `REGISTRY_NAME`   |
 | `cas.image.repository`       | Image repository                                                                                                                                                                                                                                      | `REPOSITORY_NAME` |
 | `cas.containerPorts.http`    | controlplane HTTP container port                                                                                                                                                                                                                      | `8000`            |

diff --git a/deployment/chainloop/templates/controlplane/configmap.yaml b/deployment/chainloop/templates/controlplane/configmap.yaml
@@ -41,6 +41,9 @@ data:
         addr: {{ printf "%s-api:%.0f" (include "chainloop.cas.fullname" .) (coalesce .Values.cas.serviceAPI.port .Values.cas.serviceAPI.ports.http) }}
       insecure: {{ empty (include "controlplane.tls-secret-name" .) }}
       download_url: {{ include "chainloop.cas.external_url" . }}/download
+      {{- if .Values.cas.defaultMaxEntrySize }}
+      default_entry_max_size: {{ .Values.cas.defaultMaxEntrySize | quote }}
+      {{- end }}
     plugins_dir: {{ .Values.controlplane.pluginsDir }}
     referrer_shared_index:
       {{- toYaml .Values.controlplane.referrerSharedIndex | nindent 6 }}

diff --git a/deployment/chainloop/values.yaml b/deployment/chainloop/values.yaml
@@ -927,6 +927,9 @@ cas:
   ## @param cas.replicaCount Number of replicas
   replicaCount: 2
 
+  ## @extra cas.defaultMaxEntrySize Maximum size for each entry in the CAS backend, default 100MB
+  # defaultMaxEntrySize: 100MB
+
   ## @param cas.image.registry [default: REGISTRY_NAME] Image registry
   ## @param cas.image.repository [default: REPOSITORY_NAME] Image repository
   ## @skip cas.image.tag