Add read-through caching for publisher (#113)

Signed-off-by: Igor Shishkin <me@teran.dev>

cmd/publisher/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	memcacheCli "github.com/bradfitz/gomemcache/memcache"
 	"github.com/kelseyhightower/envconfig"
 	"github.com/labstack/echo-contrib/echoprometheus"
 	echo "github.com/labstack/echo/v4"
@@ -21,6 +22,7 @@ import (
 
 	htmlPresenter "github.com/teran/archived/presenter/publisher/html"
 	awsBlobRepo "github.com/teran/archived/repositories/blob/aws"
+	"github.com/teran/archived/repositories/cache/metadata/memcache"
 	"github.com/teran/archived/repositories/metadata/postgresql"
 	"github.com/teran/archived/service"
 )
@@ -38,6 +40,9 @@ type config struct {
 
 	MetadataDSN string `envconfig:"METADATA_DSN" required:"true"`
 
+	MemcacheServers []string      `envconfig:"MEMCACHE_SERVERS"`
+	MemcacheTTL     time.Duration `envconfig:"MEMCACHE_TTL" default:"60m"`
+
 	BLOBS3Endpoint         string        `envconfig:"BLOB_S3_ENDPOINT" required:"true"`
 	BLOBS3Bucket           string        `envconfig:"BLOB_S3_BUCKET" required:"true"`
 	BLOBS3PresignedLinkTTL time.Duration `envconfig:"BLOB_S3_PRESIGNED_LINK_TTL" default:"5m"`
@@ -82,7 +87,22 @@ func main() {
 		panic(err)
 	}
 
-	postgresqlRepo := postgresql.New(db)
+	repo := postgresql.New(db)
+
+	var cli *memcacheCli.Client
+	if len(cfg.MemcacheServers) > 0 {
+		log.Debugf(
+			"%d memcache servers specified for metadata caching. Initializing read-through cache ...",
+			len(cfg.MemcacheServers),
+		)
+
+		cli = memcacheCli.New(cfg.MemcacheServers...)
+		if err := cli.Ping(); err != nil {
+			panic(err)
+		}
+
+		repo = memcache.New(cli, repo, cfg.MemcacheTTL)
+	}
 
 	awsSession, err := session.NewSession(&aws.Config{
 		Endpoint:         aws.String(cfg.BLOBS3Endpoint),
@@ -96,7 +116,7 @@ func main() {
 
 	blobRepo := awsBlobRepo.New(s3.New(awsSession), cfg.BLOBS3Bucket, cfg.BLOBS3PresignedLinkTTL)
 
-	publisherSvc := service.NewPublisher(postgresqlRepo, blobRepo, cfg.VersionsPerPage, cfg.ObjectsPerPage)
+	publisherSvc := service.NewPublisher(repo, blobRepo, cfg.VersionsPerPage, cfg.ObjectsPerPage)
 
 	p := htmlPresenter.New(publisherSvc, cfg.HTMLTemplateDir, cfg.StaticDir)
 	p.Register(e)
@@ -119,27 +139,45 @@ func main() {
 		})
 
 		http.HandleFunc("/healthz/readiness", func(w http.ResponseWriter, r *http.Request) {
+			if len(cfg.MemcacheServers) > 0 {
+				if err := cli.Ping(); err != nil {
+					log.Warnf("memcache.Ping() error on readiness probe: %s", err)
+
+					w.WriteHeader(http.StatusServiceUnavailable)
+					w.Write([]byte("failed\n"))
+					return
+				}
+			}
 			if err := db.Ping(); err != nil {
 				log.Warnf("db.Ping() error on readiness probe: %s", err)
 
 				w.WriteHeader(http.StatusServiceUnavailable)
 				w.Write([]byte("failed\n"))
-			} else {
-				w.WriteHeader(http.StatusOK)
-				w.Write([]byte("ok\n"))
+				return
 			}
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("ok\n"))
 		})
 
 		http.HandleFunc("/healthz/liveness", func(w http.ResponseWriter, r *http.Request) {
+			if len(cfg.MemcacheServers) > 0 {
+				if err := cli.Ping(); err != nil {
+					log.Warnf("memcache.Ping() error on readiness probe: %s", err)
+
+					w.WriteHeader(http.StatusServiceUnavailable)
+					w.Write([]byte("failed\n"))
+					return
+				}
+			}
 			if err := db.Ping(); err != nil {
-				log.Warnf("db.Ping() error on liveness probe: %s", err)
+				log.Warnf("db.Ping() error on readiness probe: %s", err)
 
 				w.WriteHeader(http.StatusServiceUnavailable)
 				w.Write([]byte("failed\n"))
-			} else {
-				w.WriteHeader(http.StatusOK)
-				w.Write([]byte("ok\n"))
+				return
 			}
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("ok\n"))
 		})
 
 		return http.ListenAndServe(cfg.MetricsAddr, nil)

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/Masterminds/squirrel v1.5.4
 	github.com/alecthomas/kingpin/v2 v2.4.0
 	github.com/aws/aws-sdk-go v1.55.5
+	github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874
 	github.com/go-ozzo/ozzo-validation/v4 v4.3.0
 	github.com/golang-migrate/migrate/v4 v4.17.1
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0
@@ -18,7 +19,7 @@ require (
 	github.com/prometheus/client_golang v1.19.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
-	github.com/teran/go-docker-testsuite v0.0.7
+	github.com/teran/go-docker-testsuite v0.0.8
 	github.com/teran/go-grpctest v0.0.6
 	github.com/teran/go-ptr v1.1.0
 	github.com/teran/go-time v0.0.2

go.sum

@@ -22,6 +22,8 @@ github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU
 github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 h1:N7oVaKyGp8bttX0bfZGmcGkjz7DLQXhAn3DNd3T0ous=
+github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874/go.mod h1:r5xuitiExdLAJ09PR7vBVENGvp4ZuTBeWTGtxuX3K+c=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -250,8 +252,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/teran/echo-grpc-server v0.0.2 h1:2+5zgDdRvYY2eXv4Z6o5Ku72NFKRMNAe9W3/i9YKzyo=
 github.com/teran/echo-grpc-server v0.0.2/go.mod h1:chEwqVEY4z4awjWY3+GR4iXOcB0iq0fZk/lkzdzy/nk=
-github.com/teran/go-docker-testsuite v0.0.7 h1:MJUpgCg20T2q3QSidQiE/WbaxsnHR0ZAGJr5rvKok4c=
-github.com/teran/go-docker-testsuite v0.0.7/go.mod h1:NsMzat79U8UCE51GkSPzzIsHdNPczYqReeeYizgn5H0=
+github.com/teran/go-docker-testsuite v0.0.8 h1:xNrU3XLeSnFyN4164hzTnZqnjCX6FeQH0GDjqRHPmhg=
+github.com/teran/go-docker-testsuite v0.0.8/go.mod h1:X8PV8C6ym1X5ig7cqi0Fqju7/5W0e1KTbqnNKIiExEY=
 github.com/teran/go-grpctest v0.0.6 h1:q34p64c8icQq5TFgLsDa9VM8BThhesN43XTS8PJAh38=
 github.com/teran/go-grpctest v0.0.6/go.mod h1:rNcMrEhPlNH7DFpQbcyl2vFzvfrx4UsxGuf3i/W77d0=
 github.com/teran/go-ptr v1.1.0 h1:c/5SPr+FB4zfmmKVB+KTCNKpgC/zWKBY5JHReL84lKU=