Add TTL setting for container versions in CLI, GC and manager (#269)

Reincarnation of staled PR #196 Closes #129 --------- Signed-off-by: Igor Shishkin <me@teran.ru> Co-authored-by: Демин Михаил <mddemin@nic.ru>
teran · Nov 27, 2024 · 1ab61c6 · 1ab61c6
1 parent bde668c
commit 1ab61c6
Show file tree

Hide file tree

Showing 24 changed files with 487 additions and 143 deletions.
diff --git a/README.md b/README.md
@@ -143,6 +143,9 @@ container rename <old-name> <new-name>
 container delete <name>
     delete the given container
 
+container ttl <name> <ttl>
+    set TTL (in hours) for container versions
+
 container list
     list containers
 

diff --git a/cli/service/pb_mock.go b/cli/service/pb_mock.go
@@ -2,6 +2,7 @@ package service
 
 import (
 	"context"
+	"time"
 
 	v1proto "github.com/teran/archived/manager/presenter/grpc/proto/v1"
 	"github.com/teran/archived/repositories/blob/mock"
@@ -60,6 +61,11 @@ func (m *protoClientMock) DeleteContainer(ctx context.Context, in *v1proto.Delet
 	return &v1proto.DeleteContainerResponse{}, args.Error(0)
 }
 
+func (m *protoClientMock) SetContainerParameters(ctx context.Context, in *v1proto.SetContainerParametersRequest, opts ...grpc.CallOption) (*v1proto.SetContainerParametersResponse, error) {
+	args := m.Called(in.GetNamespace(), in.GetName(), time.Duration(in.GetTtlSeconds())*time.Second)
+	return &v1proto.SetContainerParametersResponse{}, args.Error(0)
+}
+
 func (m *protoClientMock) ListContainers(ctx context.Context, in *v1proto.ListContainersRequest, opts ...grpc.CallOption) (*v1proto.ListContainersResponse, error) {
 	args := m.Called(in.GetNamespace())
 	return &v1proto.ListContainersResponse{

diff --git a/cli/service/service.go b/cli/service/service.go
@@ -5,13 +5,15 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"time"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/teran/archived/cli/service/source"
 	cache "github.com/teran/archived/cli/service/stat_cache"
 	v1proto "github.com/teran/archived/manager/presenter/grpc/proto/v1"
+	ptr "github.com/teran/go-ptr"
 )
 
 type Service interface {
@@ -20,11 +22,12 @@ type Service interface {
 	ListNamespaces() func(ctx context.Context) error
 	DeleteNamespace(namespaceName string) func(ctx context.Context) error
 
-	CreateContainer(namespaceName, containerName string) func(ctx context.Context) error
+	CreateContainer(namespaceName, containerName string, ttl time.Duration) func(ctx context.Context) error
 	MoveContainer(namespaceName, containerName, destinationNamespace string) func(ctx context.Context) error
 	RenameContainer(namespaceName, oldName, newName string) func(ctx context.Context) error
 	ListContainers(namespaceName string) func(ctx context.Context) error
 	DeleteContainer(namespaceName, containerName string) func(ctx context.Context) error
+	SetContainerParameters(namespaceName, containerName string, ttl time.Duration) func(ctx context.Context) error
 
 	CreateVersion(namespaceName, containerName string, shouldPublish bool, src source.Source) func(ctx context.Context) error
 	DeleteVersion(namespaceName, containerName, versionID string) func(ctx context.Context) error
@@ -103,11 +106,12 @@ func (s *service) DeleteNamespace(namespaceName string) func(ctx context.Context
 	}
 }
 
-func (s *service) CreateContainer(namespaceName, containerName string) func(ctx context.Context) error {
+func (s *service) CreateContainer(namespaceName, containerName string, ttl time.Duration) func(ctx context.Context) error {
 	return func(ctx context.Context) error {
 		_, err := s.cli.CreateContainer(ctx, &v1proto.CreateContainerRequest{
-			Namespace: namespaceName,
-			Name:      containerName,
+			Namespace:  namespaceName,
+			Name:       containerName,
+			TtlSeconds: ptr.Int64(int64(ttl.Seconds())),
 		})
 		if err != nil {
 			return errors.Wrap(err, "error creating container")
@@ -178,6 +182,22 @@ func (s *service) DeleteContainer(namespaceName, containerName string) func(ctx
 	}
 }
 
+func (s *service) SetContainerParameters(namespaceName, containerName string, ttl time.Duration) func(ctx context.Context) error {
+	return func(ctx context.Context) error {
+		_, err := s.cli.SetContainerParameters(ctx, &v1proto.SetContainerParametersRequest{
+			Namespace:  namespaceName,
+			Name:       containerName,
+			TtlSeconds: ptr.Int64(int64(ttl.Seconds())),
+		})
+		if err != nil {
+			return errors.Wrap(err, "error setting container versions TTL")
+		}
+
+		fmt.Printf("container `%s` versions TTL set to %s\n", containerName, ttl)
+		return nil
+	}
+}
+
 func (s *service) CreateVersion(namespaceName, containerName string, shouldPublish bool, src source.Source) func(ctx context.Context) error {
 	return func(ctx context.Context) error {
 		log.Tracef("creating version ...")

diff --git a/cli/service/service_test.go b/cli/service/service_test.go
@@ -3,6 +3,7 @@ package service
 import (
 	"context"
 	"testing"
+	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
@@ -50,7 +51,7 @@ func (s *serviceTestSuite) TestDeleteNamespace() {
 func (s *serviceTestSuite) TestCreateContainer() {
 	s.cliMock.On("CreateContainer", defaultNamespace, "test-container").Return(nil).Once()
 
-	fn := s.svc.CreateContainer(defaultNamespace, "test-container")
+	fn := s.svc.CreateContainer(defaultNamespace, "test-container", -1)
 	s.Require().NoError(fn(s.ctx))
 }
 
@@ -82,6 +83,13 @@ func (s *serviceTestSuite) TestDeleteContainer() {
 	s.Require().NoError(fn(s.ctx))
 }
 
+func (s *serviceTestSuite) TestSetContainerParameters() {
+	s.cliMock.On("SetContainerParameters", defaultNamespace, "test-container1", 3600*time.Second).Return(nil).Once()
+
+	fn := s.svc.SetContainerParameters(defaultNamespace, "test-container1", 3600*time.Second)
+	s.Require().NoError(fn(s.ctx))
+}
+
 func (s *serviceTestSuite) TestCreateVersion() {
 	s.cliMock.On("CreateVersion", defaultNamespace, "container1").Return("version_id", nil).Once()
 

diff --git a/cmd/cli/main.go b/cmd/cli/main.go
@@ -83,6 +83,7 @@ var (
 	container           = app.Command("container", "container operations")
 	containerCreate     = container.Command("create", "create new container")
 	containerCreateName = containerCreate.Arg("name", "name of the container to create").Required().String()
+	containerCreateTTL  = containerCreate.Flag("ttl", "Default container TTL").Default("-1ns").Duration()
 
 	containerMove          = container.Command("move", "move container to another namespace")
 	containerMoveName      = containerMove.Arg("name", "container namespace to move").Required().String()
@@ -95,6 +96,10 @@ var (
 	containerDelete     = container.Command("delete", "delete the given container")
 	containerDeleteName = containerDelete.Arg("name", "name of the container to delete").Required().String()
 
+	containerSet          = container.Command("set", "set parameters for container")
+	containerSetContainer = containerSet.Arg("name", "name of the container").Required().String()
+	containerSetTTL       = containerSet.Flag("ttl", "Container TTL").Default("-1ns").Duration()
+
 	containerList = container.Command("list", "list containers")
 
 	version                = app.Command("version", "version operations")
@@ -233,11 +238,12 @@ func main() {
 	r.Register(namespaceList.FullCommand(), cliSvc.ListNamespaces())
 	r.Register(namespaceDelete.FullCommand(), cliSvc.DeleteNamespace(*namespaceDeleteName))
 
-	r.Register(containerCreate.FullCommand(), cliSvc.CreateContainer(*namespaceName, *containerCreateName))
+	r.Register(containerCreate.FullCommand(), cliSvc.CreateContainer(*namespaceName, *containerCreateName, *containerCreateTTL))
 	r.Register(containerMove.FullCommand(), cliSvc.MoveContainer(*namespaceName, *containerMoveName, *containerMoveNamespace))
 	r.Register(containerRename.FullCommand(), cliSvc.RenameContainer(*namespaceName, *containerRenameOldName, *containerRenameNewName))
 	r.Register(containerList.FullCommand(), cliSvc.ListContainers(*namespaceName))
 	r.Register(containerDelete.FullCommand(), cliSvc.DeleteContainer(*namespaceName, *containerDeleteName))
+	r.Register(containerSet.FullCommand(), cliSvc.SetContainerParameters(*namespaceName, *containerSetContainer, *containerSetTTL))
 
 	r.Register(versionList.FullCommand(), cliSvc.ListVersions(*namespaceName, *versionListContainer))
 	r.Register(versionCreate.FullCommand(), cliSvc.CreateVersion(
@@ -249,7 +255,6 @@ func main() {
 	r.Register(objectList.FullCommand(), cliSvc.ListObjects(*namespaceName, *objectListContainer, *objectListVersion))
 	r.Register(objectURL.FullCommand(), cliSvc.GetObjectURL(*namespaceName, *objectURLContainer, *objectURLVersion, *objectURLKey))
 	r.Register(deleteObject.FullCommand(), cliSvc.DeleteObject(*namespaceName, *deleteObjectContainer, *deleteObjectVersion, *deleteObjectKey))
-
 	r.Register(statCacheShowPath.FullCommand(), func(ctx context.Context) error {
 		fmt.Println(*cacheDir)
 		return nil

diff --git a/cmd/seeder/main.go b/cmd/seeder/main.go
@@ -110,7 +110,7 @@ func main() {
 		}
 		for j := 0; j <= cfg.CreateContainersPerNamespace; j++ {
 			container := fmt.Sprintf("container-%06d", j)
-			if err := managerSvc.CreateContainer(ctx, namespace, container); err != nil {
+			if err := managerSvc.CreateContainer(ctx, namespace, container, -1); err != nil {
 				panic(err)
 			}
 			for k := 0; k <= cfg.CreateVersionsPerContainer; k++ {