diff --git a/models/blob.go b/models/blob.go new file mode 100644 index 0000000..1bd05fb --- /dev/null +++ b/models/blob.go @@ -0,0 +1,7 @@ +package models + +type Blob struct { + Checksum string + Size uint64 + MimeType string +} diff --git a/repositories/blob/aws/aws.go b/repositories/blob/aws/aws.go index 9964f1b..2c4c5f3 100644 --- a/repositories/blob/aws/aws.go +++ b/repositories/blob/aws/aws.go @@ -2,6 +2,8 @@ package aws import ( "context" + "net/url" + "path" "time" "github.com/aws/aws-sdk-go/aws" @@ -40,10 +42,12 @@ func (s *s3driver) PutBlobURL(ctx context.Context, key string) (string, error) { return url, nil } -func (s *s3driver) GetBlobURL(ctx context.Context, key string) (string, error) { +func (s *s3driver) GetBlobURL(ctx context.Context, key, mimeType, filename string) (string, error) { req, _ := s.cli.GetObjectRequest(&s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(key), + Bucket: aws.String(s.bucket), + Key: aws.String(key), + ResponseContentType: aws.String(mimeType), + ResponseContentDisposition: aws.String("attachment; filename=" + url.QueryEscape(path.Base(filename))), }) url, err := req.Presign(s.ttl) diff --git a/repositories/blob/aws/aws_test.go b/repositories/blob/aws/aws_test.go index 81c2757..c9a178d 100644 --- a/repositories/blob/aws/aws_test.go +++ b/repositories/blob/aws/aws_test.go @@ -25,11 +25,13 @@ func (s *repoTestSuite) TestAll() { err = uploadToURL(s.ctx, url, []byte("test data")) s.Require().NoError(err) - url, err = s.driver.GetBlobURL(s.ctx, "blah/test/key.txt") + url, err = s.driver.GetBlobURL(s.ctx, "blah/test/key.txt", "application/json", "test-file.txt") s.Require().NoError(err) - data, err := fetchURL(s.ctx, url) + data, mimeType, disposition, err := fetchURL(s.ctx, url) s.Require().NoError(err) + s.Require().Equal("application/json", mimeType) + s.Require().Equal("attachment; filename=test-file.txt", disposition) s.Require().Equal("test data", string(data)) } @@ -106,19 +108,20 @@ func TestRepoTestSuite(t *testing.T) { suite.Run(t, &repoTestSuite{}) } -func fetchURL(ctx context.Context, url string) ([]byte, error) { +func fetchURL(ctx context.Context, url string) ([]byte, string, string, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return nil, err + return nil, "", "", err } resp, err := http.DefaultClient.Do(req) if err != nil { - return nil, err + return nil, "", "", err } defer resp.Body.Close() - return io.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) + return data, resp.Header.Get("Content-Type"), resp.Header.Get("Content-Disposition"), err } func uploadToURL(ctx context.Context, url string, payload []byte) error { diff --git a/repositories/blob/blob.go b/repositories/blob/blob.go index 94a20f0..c9e3798 100644 --- a/repositories/blob/blob.go +++ b/repositories/blob/blob.go @@ -6,5 +6,5 @@ import ( type Repository interface { PutBlobURL(ctx context.Context, key string) (string, error) - GetBlobURL(ctx context.Context, key string) (string, error) + GetBlobURL(ctx context.Context, key, mimeType, filename string) (string, error) } diff --git a/repositories/blob/mock/mock.go b/repositories/blob/mock/mock.go index 39a9b0f..0a4be9f 100644 --- a/repositories/blob/mock/mock.go +++ b/repositories/blob/mock/mock.go @@ -23,7 +23,7 @@ func (m *Mock) PutBlobURL(_ context.Context, key string) (string, error) { return args.String(0), args.Error(1) } -func (m *Mock) GetBlobURL(_ context.Context, key string) (string, error) { - args := m.Called(key) +func (m *Mock) GetBlobURL(_ context.Context, key, mimeType, filename string) (string, error) { + args := m.Called(key, mimeType, filename) return args.String(0), args.Error(1) } diff --git a/repositories/cache/metadata/memcache/memcache.go b/repositories/cache/metadata/memcache/memcache.go index bce23bc..f77143f 100644 --- a/repositories/cache/metadata/memcache/memcache.go +++ b/repositories/cache/metadata/memcache/memcache.go @@ -492,6 +492,50 @@ func (m *memcache) GetBlobKeyByObject(ctx context.Context, namespace, container, return retrievedValue, nil } +func (m *memcache) GetBlobByObject(ctx context.Context, namespace, container, version, key string) (models.Blob, error) { + cacheKey := strings.Join([]string{ + m.keyPrefix, + "GetBlobByObject", + namespace, + container, + version, + key, + }, ":") + + item, err := m.cli.Get(cacheKey) + if err != nil { + if errors.Is(err, memcacheCli.ErrCacheMiss) { + log.WithFields(log.Fields{ + "key": cacheKey, + }).Tracef("cache miss") + + blob, err := m.repo.GetBlobByObject(ctx, namespace, container, version, key) + if err != nil { + return models.Blob{}, err + } + + if err = store(m, cacheKey, blob); err != nil { + return models.Blob{}, err + } + + return blob, err + } + + return models.Blob{}, err + } + log.WithFields(log.Fields{ + "key": cacheKey, + }).Tracef("cache hit") + + var retrievedValue models.Blob + err = json.Unmarshal(item.Value, &retrievedValue) + if err != nil { + return models.Blob{}, err + } + + return retrievedValue, nil +} + func (m *memcache) EnsureBlobKey(ctx context.Context, key string, size uint64) error { return m.repo.EnsureBlobKey(ctx, key, size) } diff --git a/repositories/cache/metadata/memcache/memcache_test.go b/repositories/cache/metadata/memcache/memcache_test.go index b077bf1..1e932f5 100644 --- a/repositories/cache/metadata/memcache/memcache_test.go +++ b/repositories/cache/metadata/memcache/memcache_test.go @@ -186,6 +186,30 @@ func (s *memcacheTestSuite) TestGetBlobKeyByObject() { s.Require().Equal("deadbeef", casKey) } +func (s *memcacheTestSuite) TestGetBlobByObject() { + s.repoMock.On("GetBlobByObject", defaultNamespace, "container", "version", "key").Return(models.Blob{ + Checksum: "deadbeef", + Size: 1234, + MimeType: "application/x-gzip", + }, nil).Once() + + blob, err := s.cache.GetBlobByObject(s.ctx, defaultNamespace, "container", "version", "key") + s.Require().NoError(err) + s.Require().Equal(models.Blob{ + Checksum: "deadbeef", + Size: 1234, + MimeType: "application/x-gzip", + }, blob) + + blob, err = s.cache.GetBlobByObject(s.ctx, defaultNamespace, "container", "version", "key") + s.Require().NoError(err) + s.Require().Equal(models.Blob{ + Checksum: "deadbeef", + Size: 1234, + MimeType: "application/x-gzip", + }, blob) +} + func (s *memcacheTestSuite) TestGetBlobKeyByObjectError() { s.repoMock.On("GetBlobKeyByObject", defaultNamespace, "container", "version", "key").Return("", errors.New("some error")).Once() diff --git a/repositories/metadata/metadata.go b/repositories/metadata/metadata.go index 897a851..16b1f2b 100644 --- a/repositories/metadata/metadata.go +++ b/repositories/metadata/metadata.go @@ -41,7 +41,8 @@ type Repository interface { RemapObject(ctx context.Context, namespace, container, version, key, newCASKey string) error CreateBLOB(ctx context.Context, checksum string, size uint64, mimeType string) error - GetBlobKeyByObject(ctx context.Context, namespace, scontainer, version, key string) (string, error) + GetBlobKeyByObject(ctx context.Context, namespace, container, version, key string) (string, error) + GetBlobByObject(ctx context.Context, namespace, container, version, key string) (models.Blob, error) EnsureBlobKey(ctx context.Context, key string, size uint64) error CountStats(ctx context.Context) (*emodels.Stats, error) diff --git a/repositories/metadata/mock/mock.go b/repositories/metadata/mock/mock.go index f7ce8c7..2b0200b 100644 --- a/repositories/metadata/mock/mock.go +++ b/repositories/metadata/mock/mock.go @@ -134,6 +134,11 @@ func (m *Mock) GetBlobKeyByObject(_ context.Context, namespace, container, versi return args.String(0), args.Error(1) } +func (m *Mock) GetBlobByObject(_ context.Context, namespace, container, version, key string) (models.Blob, error) { + args := m.Called(namespace, container, version, key) + return args.Get(0).(models.Blob), args.Error(1) +} + func (m *Mock) EnsureBlobKey(_ context.Context, key string, size uint64) error { args := m.Called(key, size) return args.Error(0) diff --git a/repositories/metadata/postgresql/blobs.go b/repositories/metadata/postgresql/blobs.go index 887e1e1..90a069d 100644 --- a/repositories/metadata/postgresql/blobs.go +++ b/repositories/metadata/postgresql/blobs.go @@ -4,6 +4,7 @@ import ( "context" sq "github.com/Masterminds/squirrel" + "github.com/teran/archived/models" ) func (r *repository) CreateBLOB(ctx context.Context, checksum string, size uint64, mimeType string) error { @@ -52,6 +53,38 @@ func (r *repository) GetBlobKeyByObject(ctx context.Context, namespace, containe return checksum, nil } +func (r *repository) GetBlobByObject(ctx context.Context, namespace, container, version, key string) (models.Blob, error) { + row, err := selectQueryRow(ctx, r.db, psql. + Select( + "b.checksum AS checksum", + "b.size AS size", + "b.mime_type AS mime_type", + ). + From("blobs b"). + Join("objects o ON o.blob_id = b.id"). + Join("object_keys ok ON ok.id = o.key_id"). + Join("versions v ON o.version_id = v.id"). + Join("containers c ON v.container_id = c.id"). + Join("namespaces ns ON c.namespace_id = ns.id"). + Where(sq.Eq{ + "ns.name": namespace, + "c.name": container, + "v.name": version, + "ok.key": key, + "v.is_published": true, + })) + if err != nil { + return models.Blob{}, mapSQLErrors(err) + } + + var b models.Blob + if err := row.Scan(&b.Checksum, &b.Size, &b.MimeType); err != nil { + return models.Blob{}, mapSQLErrors(err) + } + + return b, nil +} + func (r *repository) EnsureBlobKey(ctx context.Context, key string, size uint64) error { row, err := selectQueryRow(ctx, r.db, psql. Select("id"). diff --git a/repositories/metadata/postgresql/blobs_test.go b/repositories/metadata/postgresql/blobs_test.go index 44d33c0..99ffedc 100644 --- a/repositories/metadata/postgresql/blobs_test.go +++ b/repositories/metadata/postgresql/blobs_test.go @@ -1,6 +1,9 @@ package postgresql -import "github.com/teran/archived/repositories/metadata" +import ( + "github.com/teran/archived/models" + "github.com/teran/archived/repositories/metadata" +) func (s *postgreSQLRepositoryTestSuite) TestBlobs() { const ( @@ -32,6 +35,14 @@ func (s *postgreSQLRepositoryTestSuite) TestBlobs() { casKey, err := s.repo.GetBlobKeyByObject(s.ctx, defaultNamespace, containerName, versionID, "test-object.txt") s.Require().NoError(err) s.Require().Equal(checksum, casKey) + + blob, err := s.repo.GetBlobByObject(s.ctx, defaultNamespace, containerName, versionID, "test-object.txt") + s.Require().NoError(err) + s.Require().Equal(models.Blob{ + Checksum: checksum, + Size: 15, + MimeType: "text/plain", + }, blob) } func (s *postgreSQLRepositoryTestSuite) TestGetBlobKeyByObjectErrors() { @@ -69,6 +80,41 @@ func (s *postgreSQLRepositoryTestSuite) TestGetBlobKeyByObjectErrors() { s.Require().Equal(metadata.ErrNotFound, err) } +func (s *postgreSQLRepositoryTestSuite) TestGetBlobByObjectErrors() { + s.tp.On("Now").Return("2024-01-02T01:02:03Z").Twice() + + // Nothing exists: container, version, key + _, err := s.repo.GetBlobByObject(s.ctx, defaultNamespace, "container", "version", "key") + s.Require().Error(err) + s.Require().Equal(metadata.ErrNotFound, err) + + // version & key doesn't exist + err = s.repo.CreateContainer(s.ctx, defaultNamespace, "container") + s.Require().NoError(err) + + _, err = s.repo.GetBlobByObject(s.ctx, defaultNamespace, "container", "version", "key") + s.Require().Error(err) + s.Require().Equal(metadata.ErrNotFound, err) + + // version is unpublished & key doesn't exist + s.tp.On("Now").Return("2024-01-02T01:02:03Z").Once() + + version, err := s.repo.CreateVersion(s.ctx, defaultNamespace, "container") + s.Require().NoError(err) + + _, err = s.repo.GetBlobByObject(s.ctx, defaultNamespace, "container", version, "key") + s.Require().Error(err) + s.Require().Equal(metadata.ErrNotFound, err) + + // version is published but key doesn't exist + err = s.repo.MarkVersionPublished(s.ctx, defaultNamespace, "container", version) + s.Require().NoError(err) + + _, err = s.repo.GetBlobByObject(s.ctx, defaultNamespace, "container", version, "key") + s.Require().Error(err) + s.Require().Equal(metadata.ErrNotFound, err) +} + func (s *postgreSQLRepositoryTestSuite) TestEnsureBlobKey() { s.tp.On("Now").Return("2024-01-02T01:02:03Z").Once() diff --git a/service/service.go b/service/service.go index 1234628..6b551fa 100644 --- a/service/service.go +++ b/service/service.go @@ -248,12 +248,12 @@ func (s *service) GetObjectURL(ctx context.Context, namespace, container, versio } } - objectKey, err := s.mdRepo.GetBlobKeyByObject(ctx, namespace, container, versionID, key) + blob, err := s.mdRepo.GetBlobByObject(ctx, namespace, container, versionID, key) if err != nil { return "", mapMetadataErrors(err) } - return s.blobRepo.GetBlobURL(ctx, objectKey) + return s.blobRepo.GetBlobURL(ctx, blob.Checksum, blob.MimeType, key) } func (s *service) EnsureBLOBPresenceOrGetUploadURL(ctx context.Context, checksum string, size uint64, mimeType string) (string, error) { diff --git a/service/service_test.go b/service/service_test.go index a6daf00..bd23efb 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -257,8 +257,12 @@ func (s *serviceTestSuite) TestListObjects() { func (s *serviceTestSuite) TestGetObjectURL() { // Happy path - s.mdRepoMock.On("GetBlobKeyByObject", defaultNamespace, "container", "versionID", "key").Return("deadbeef", nil).Once() - s.blobRepoMock.On("GetBlobURL", "deadbeef").Return("url", nil).Once() + s.mdRepoMock.On("GetBlobByObject", defaultNamespace, "container", "versionID", "key").Return(models.Blob{ + Checksum: "deadbeef", + Size: 1234, + MimeType: "application/json", + }, nil).Once() + s.blobRepoMock.On("GetBlobURL", "deadbeef", "application/json", "key").Return("url", nil).Once() url, err := s.svc.GetObjectURL(s.ctx, defaultNamespace, "container", "versionID", "key") s.Require().NoError(err) @@ -302,8 +306,12 @@ func (s *serviceTestSuite) TestListObjectsErrNotFound() { func (s *serviceTestSuite) TestGetObjectURLWithLatestVersion() { s.mdRepoMock.On("GetLatestPublishedVersionByContainer", defaultNamespace, "container12").Return("versionID", nil).Once() - s.mdRepoMock.On("GetBlobKeyByObject", defaultNamespace, "container12", "versionID", "key").Return("deadbeef", nil).Once() - s.blobRepoMock.On("GetBlobURL", "deadbeef").Return("url", nil).Once() + s.mdRepoMock.On("GetBlobByObject", defaultNamespace, "container12", "versionID", "key").Return(models.Blob{ + Checksum: "deadbeef", + Size: 1234, + MimeType: "application/json", + }, nil).Once() + s.blobRepoMock.On("GetBlobURL", "deadbeef", "application/json", "key").Return("url", nil).Once() url, err := s.svc.GetObjectURL(s.ctx, defaultNamespace, "container12", "latest", "key") s.Require().NoError(err)