Add PostgreSQL metadata repository

go.mod

@@ -2,11 +2,66 @@ module github.com/teran/archived
 
 go 1.22.5
 
-require github.com/stretchr/testify v1.9.0
+require (
+	github.com/Masterminds/squirrel v1.5.4
+	github.com/golang-migrate/migrate/v4 v4.17.1
+	github.com/lib/pq v1.10.9
+	github.com/pkg/errors v0.9.1
+	github.com/stretchr/testify v1.9.0
+	github.com/teran/go-docker-testsuite v0.0.6
+	github.com/teran/go-ptr v1.1.0
+	github.com/teran/go-time v0.0.2
+)
 
 require (
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/cilium/ebpf v0.11.0 // indirect
+	github.com/containerd/cgroups/v3 v3.0.3 // indirect
+	github.com/containerd/containerd v1.7.16 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/docker v27.0.0+incompatible // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
+	github.com/jackc/pgconn v1.14.3 // indirect
+	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.3 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgtype v1.14.0 // indirect
+	github.com/jackc/pgx/v4 v4.18.3 // indirect
+	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
+	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/mountinfo v0.7.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/opencontainers/runtime-spec v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
+	github.com/teran/go-random v0.0.1 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect
+	go.opentelemetry.io/otel v1.26.0 // indirect
+	go.opentelemetry.io/otel/metric v1.26.0 // indirect
+	go.opentelemetry.io/otel/trace v1.26.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	golang.org/x/tools v0.10.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

repositories/metadata/metadata.go

@@ -13,7 +13,7 @@ type Repository interface {
 	MarkVersionPublished(ctx context.Context, container, version string) error
 
 	CreateObject(ctx context.Context, container, version, key, casKey string) error
-	ListObjects(ctx context.Context, container, version, key string) ([]string, error)
+	ListObjects(ctx context.Context, container, version string, offset, limit uint64) ([]string, error)
 	DeleteObject(ctx context.Context, container, version, key string) error
 	RemapObject(ctx context.Context, container, version, key, newCASKey string) error
 

repositories/metadata/postgresql/blobs.go

@@ -0,0 +1,26 @@
+package postgresql
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+)
+
+func (r *repository) CreateBLOB(ctx context.Context, checksum string, size uint64, mimeType string) error {
+	_, err := psql.
+		Insert("blobs").
+		Columns(
+			"checksum",
+			"size",
+			"mime_type",
+		).
+		Values(
+			checksum,
+			size,
+			mimeType,
+		).
+		RunWith(r.db).
+		ExecContext(ctx)
+
+	return errors.Wrap(err, "error executing SQL query")
+}

repositories/metadata/postgresql/blobs_test.go

@@ -0,0 +1,6 @@
+package postgresql
+
+func (s *postgreSQLRepositoryTestSuite) TestBlobs() {
+	err := s.repo.CreateBLOB(s.ctx, "deadbeef", 15, "text/plain")
+	s.Require().NoError(err)
+}

repositories/metadata/postgresql/containers.go

@@ -0,0 +1,58 @@
+package postgresql
+
+import (
+	"context"
+
+	sq "github.com/Masterminds/squirrel"
+	"github.com/pkg/errors"
+)
+
+func (r *repository) CreateContainer(ctx context.Context, name string) error {
+	_, err := psql.
+		Insert("containers").
+		Columns(
+			"name",
+		).
+		Values(
+			name,
+		).
+		RunWith(r.db).
+		ExecContext(ctx)
+
+	return errors.Wrap(err, "error executing SQL query")
+}
+
+func (r *repository) ListContainers(ctx context.Context) ([]string, error) {
+	rows, err := psql.
+		Select("name").
+		From("containers").
+		OrderBy("id").
+		RunWith(r.db).
+		QueryContext(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "error executing SQL query")
+	}
+	defer rows.Close()
+
+	result := []string{}
+	for rows.Next() {
+		var r string
+		if err := rows.Scan(&r); err != nil {
+			return nil, errors.Wrap(err, "error decoding database result")
+		}
+
+		result = append(result, r)
+	}
+
+	return result, nil
+}
+
+func (r *repository) DeleteContainer(ctx context.Context, name string) error {
+	_, err := psql.
+		Delete("containers").
+		Where(sq.Eq{"name": name}).
+		RunWith(r.db).
+		ExecContext(ctx)
+
+	return errors.Wrap(err, "error executing SQL query")
+}

repositories/metadata/postgresql/containers_test.go

@@ -0,0 +1,36 @@
+package postgresql
+
+func (s *postgreSQLRepositoryTestSuite) TestContainerOperations() {
+	list, err := s.repo.ListContainers(s.ctx)
+	s.Require().NoError(err)
+	s.Require().Equal([]string{}, list)
+
+	err = s.repo.CreateContainer(s.ctx, "test-container")
+	s.Require().NoError(err)
+
+	err = s.repo.CreateContainer(s.ctx, "test-container2")
+	s.Require().NoError(err)
+
+	err = s.repo.CreateContainer(s.ctx, "test-container")
+	s.Require().Error(err)
+	s.Require().Equal(
+		`error executing SQL query: pq: duplicate key value violates unique constraint "containers_name_key"`,
+		err.Error(),
+	)
+
+	list, err = s.repo.ListContainers(s.ctx)
+	s.Require().NoError(err)
+	s.Require().Equal([]string{
+		"test-container",
+		"test-container2",
+	}, list)
+
+	err = s.repo.DeleteContainer(s.ctx, "test-container")
+	s.Require().NoError(err)
+
+	list, err = s.repo.ListContainers(s.ctx)
+	s.Require().NoError(err)
+	s.Require().Equal([]string{
+		"test-container2",
+	}, list)
+}

repositories/metadata/postgresql/migrations/migrations.go

@@ -0,0 +1,37 @@
+package migrations
+
+import (
+	"database/sql"
+
+	migrate "github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/postgres"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+	_ "github.com/lib/pq"
+	"github.com/pkg/errors"
+)
+
+func MigrateUp(dsn string) error {
+	db, err := sql.Open("postgres", dsn)
+	if err != nil {
+		return errors.Wrap(err, "error opening database connection")
+	}
+	defer db.Close()
+
+	if err := db.Ping(); err != nil {
+		return errors.Wrap(err, "error pinging database")
+	}
+
+	driver, err := postgres.WithInstance(db, &postgres.Config{})
+	if err != nil {
+		return errors.Wrap(err, "error creating database instance")
+	}
+
+	m, err := migrate.NewWithDatabaseInstance(
+		"file://migrations/sql",
+		"postgres", driver)
+	if err != nil {
+		return errors.Wrap(err, "error creating migrator instance")
+	}
+
+	return errors.Wrap(m.Up(), "error migrating database")
+}

repositories/metadata/postgresql/migrations/sql/0000_initial.up.sql

@@ -0,0 +1,42 @@
+BEGIN;
+
+CREATE TABLE containers (
+    id SERIAL PRIMARY KEY,
+    name VARCHAR(255) NOT NULL
+);
+
+CREATE TABLE versions (
+    id SERIAL PRIMARY KEY,
+    container_id INT NOT NULL,
+    name VARCHAR(64) NOT NULL,
+    is_published BOOLEAN NOT NULL
+);
+
+CREATE TABLE blobs (
+    id SERIAL PRIMARY KEY,
+    checksum CHAR(64) NOT NULL,
+    size INT NOT NULL,
+    mime_type VARCHAR(64) NOT NULL
+);
+
+CREATE TABLE objects (
+    id SERIAL PRIMARY KEY,
+    version_id INT NOT NULL,
+    key VARCHAR(255) NOT NULL,
+    blob_id INT NOT NULL
+);
+
+CREATE UNIQUE INDEX containers_name_key ON containers (name);
+
+ALTER TABLE versions ADD FOREIGN KEY (container_id) REFERENCES containers (id);
+CREATE UNIQUE INDEX versions_container_id_name_key ON versions (container_id, name);
+CREATE INDEX versions_is_published_idx ON versions (is_published);
+
+ALTER TABLE objects ADD FOREIGN KEY (version_id) REFERENCES versions (id);
+CREATE UNIQUE INDEX objects_version_id_key_key ON objects (version_id, key);
+CREATE INDEX objects_key_idx ON objects (key);
+ALTER TABLE objects ADD FOREIGN KEY (blob_id) REFERENCES blobs (id);
+
+CREATE UNIQUE INDEX blobs_checksum_key ON blobs (checksum);
+
+COMMIT;