CBG-4342: [3.2.2 backport] add updated at field for persisted configs (…

…#7274)

* CBG-4336: add updated at field for persisted configs

* change time to utc

* fix default collection test issues

* flakyness with tests

* add delay to other tests too

* updates to add created_at field to configs

* remove unnecessary method

auth/auth.go

@@ -631,6 +631,8 @@ func (auth *Authenticator) UpdateUserEmail(u User, email string) error {
 		if err != nil {
 			return nil, err
 		}
+		currentUser.SetUpdatedAt()
+
 		return currentUser, nil
 	}
 
@@ -662,6 +664,7 @@ func (auth *Authenticator) rehashPassword(user User, password string) error {
 			if err != nil {
 				return nil, err
 			}
+			currentUserImpl.SetUpdatedAt()
 			return currentUserImpl, nil
 		} else {
 			return nil, base.ErrUpdateCancel
@@ -740,6 +743,7 @@ func (auth *Authenticator) DeleteRole(role Role, purge bool, deleteSeq uint64) e
 		}
 		p.setDeleted(true)
 		p.SetSequence(deleteSeq)
+		p.SetUpdatedAt()
 
 		// Update channel history for default collection
 		channelHistory := auth.calculateHistory(p.Name(), deleteSeq, p.Channels(), nil, p.ChannelHistory())
@@ -953,6 +957,8 @@ func (auth *Authenticator) RegisterNewUser(username, email string) (User, error)
 			base.WarnfCtx(auth.LogCtx, "Skipping SetEmail for user %q - Invalid email address provided: %q", base.UD(username), base.UD(email))
 		}
 	}
+	user.SetUpdatedAt()
+	user.SetCreatedAt(time.Now().UTC())
 
 	err = auth.Save(user)
 	if base.IsCasMismatch(err) {

auth/principal.go

@@ -52,6 +52,12 @@ type Principal interface {
 	setDeleted(bool)
 	IsDeleted() bool
 
+	// Sets the updated time for the principal document
+	SetUpdatedAt()
+
+	// Sets the created time for the principal document
+	SetCreatedAt(t time.Time)
+
 	// Principal includes the PrincipalCollectionAccess interface for operations against
 	// the _default._default collection (stored directly on the principal for backward
 	// compatibility)

auth/role.go

@@ -33,6 +33,8 @@ type roleImpl struct {
 	ChannelInvalSeq   uint64                                  `json:"channel_inval_seq,omitempty"` // Sequence at which the channels were invalidated. Data remains in Channels_ for history calculation.
 	Deleted           bool                                    `json:"deleted,omitempty"`
 	CollectionsAccess map[string]map[string]*CollectionAccess `json:"collection_access,omitempty"` // Nested maps of CollectionAccess, indexed by scope and collection name
+	UpdatedAt         time.Time                               `json:"updated_at"`
+	CreatedAt         time.Time                               `json:"created_at"`
 	cas               uint64
 	docID             string // key used to store the roleImpl
 }
@@ -277,6 +279,14 @@ func (role *roleImpl) Name() string {
 	return role.Name_
 }
 
+func (role *roleImpl) SetUpdatedAt() {
+	role.UpdatedAt = time.Now().UTC()
+}
+
+func (role *roleImpl) SetCreatedAt(t time.Time) {
+	role.CreatedAt = t
+}
+
 func (role *roleImpl) Sequence() uint64 {
 	return role.Sequence_
 }

db/sg_replicate_cfg.go

@@ -115,6 +115,8 @@ type ReplicationConfig struct {
 	Adhoc                  bool                      `json:"adhoc,omitempty"`
 	BatchSize              int                       `json:"batch_size,omitempty"`
 	RunAs                  string                    `json:"run_as,omitempty"`
+	UpdatedAt              *time.Time                `json:"updated_at,omitempty"`
+	CreatedAt              *time.Time                `json:"created_at,omitempty"`
 }
 
 func DefaultReplicationConfig() ReplicationConfig {
@@ -335,6 +337,9 @@ func (rc *ReplicationConfig) Upsert(ctx context.Context, c *ReplicationUpsertCon
 		rc.RunAs = *c.RunAs
 	}
 
+	timeNow := time.Now().UTC()
+	rc.UpdatedAt = &timeNow
+
 	if c.QueryParams != nil {
 		// QueryParams can be either []interface{} or map[string]interface{}, so requires type-specific copying
 		// avoid later mutating c.QueryParams
@@ -1106,6 +1111,8 @@ func (m *sgReplicateManager) UpsertReplication(ctx context.Context, replication
 		} else {
 			// Add a new replication to the cfg.  Set targetState based on initialState when specified.
 			replicationConfig := DefaultReplicationConfig()
+			createdAt := time.Now().UTC()
+			replicationConfig.CreatedAt = &createdAt
 			replicationConfig.ID = replication.ID
 			targetState := ReplicationStateRunning
 			if replication.InitialState != nil && *replication.InitialState == ReplicationStateStopped {

db/sg_replicate_cfg_test.go

@@ -536,6 +536,7 @@ func TestUpsertReplicationConfig(t *testing.T) {
 	for _, testCase := range testCases {
 		t.Run(fmt.Sprintf("%s", testCase.name), func(t *testing.T) {
 			testCase.existingConfig.Upsert(base.TestCtx(t), testCase.updatedConfig)
+			testCase.existingConfig.UpdatedAt = nil // remove updated at field for comparison below
 			equal, err := testCase.existingConfig.Equals(testCase.expectedConfig)
 			assert.NoError(t, err)
 			assert.True(t, equal)

db/users.go

@@ -86,6 +86,7 @@ func (dbc *DatabaseContext) UpdatePrincipal(ctx context.Context, updates *auth.P
 			if err != nil {
 				return replaced, princ, fmt.Errorf("Error creating user/role: %w", err)
 			}
+			princ.SetCreatedAt(time.Now().UTC())
 			changed = true
 		} else if !allowReplace {
 			err = base.HTTPErrorf(http.StatusConflict, "Already exists")
@@ -214,6 +215,7 @@ func (dbc *DatabaseContext) UpdatePrincipal(ctx context.Context, updates *auth.P
 				user.SetJWTLastUpdated(time.Now())
 			}
 		}
+		princ.SetUpdatedAt()
 		err = authenticator.Save(princ)
 		// On cas error, retry.  Otherwise break out of loop
 		if base.IsCasMismatch(err) {

rest/api_collections_test.go

@@ -1001,6 +1001,8 @@ func TestRuntimeConfigUpdateAfterConfigUpdateConflict(t *testing.T) {
 	delete(scopesConfig[scope].Collections, collection1)
 	assert.Equal(t, scopesConfig, dbCfg.Scopes)
 	originalDBCfg.Server = nil
+	dbCfg.UpdatedAt = nil // originalDBCfg fetch is from memory so has no update/create at time
+	dbCfg.CreatedAt = nil
 	assert.Equal(t, originalDBCfg, dbCfg)
 
 	// now assert that _config shows the same

rest/config.go

@@ -193,6 +193,8 @@ type DbConfig struct {
 	ChangesRequestPlus               *bool                            `json:"changes_request_plus,omitempty"`                 // If set, is used as the default value of request_plus for non-continuous replications
 	CORS                             *auth.CORSConfig                 `json:"cors,omitempty"`                                 // Per-database CORS config
 	Logging                          *DbLoggingConfig                 `json:"logging,omitempty"`                              // Per-database Logging config
+	UpdatedAt                        *time.Time                       `json:"updated_at,omitempty"`                           // Time at which the database config was last updated
+	CreatedAt                        *time.Time                       `json:"created_at,omitempty"`                           // Time at which the database config was created
 }
 
 type ScopesConfig map[string]ScopeConfig

rest/config_database_test.go

@@ -9,10 +9,14 @@
 package rest
 
 import (
+	"encoding/json"
+	"net/http"
 	"testing"
 	"time"
 
+	"github.com/couchbase/sync_gateway/base"
 	"github.com/couchbase/sync_gateway/db"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -22,3 +26,59 @@ func TestDefaultDbConfig(t *testing.T) {
 	compactIntervalDays := *(DefaultDbConfig(&sc, useXattrs).CompactIntervalDays)
 	require.Equal(t, db.DefaultCompactInterval, time.Duration(compactIntervalDays)*time.Hour*24)
 }
+
+func TestDbConfigUpdatedAtField(t *testing.T) {
+	b := base.GetTestBucket(t)
+	rt := NewRestTester(t, &RestTesterConfig{
+		CustomTestBucket: b,
+		PersistentConfig: true,
+	})
+	defer rt.Close()
+	ctx := base.TestCtx(t)
+
+	dbConfig := rt.NewDbConfig()
+	RequireStatus(t, rt.CreateDatabase("db1", dbConfig), http.StatusCreated)
+
+	sc := rt.ServerContext()
+
+	resp := rt.SendAdminRequest(http.MethodGet, "/db1/_config", "")
+	RequireStatus(t, resp, http.StatusOK)
+	var unmarshaledConfig DbConfig
+	require.NoError(t, json.Unmarshal(resp.BodyBytes(), &unmarshaledConfig))
+
+	registry := &GatewayRegistry{}
+	bName := b.GetName()
+	_, err := sc.BootstrapContext.Connection.GetMetadataDocument(ctx, bName, base.SGRegistryKey, registry)
+	require.NoError(t, err)
+
+	// Check that the config has an updatedAt field
+	require.NotNil(t, unmarshaledConfig.UpdatedAt)
+	require.NotNil(t, unmarshaledConfig.CreatedAt)
+	currUpdatedTime := unmarshaledConfig.UpdatedAt
+	currCreatedTime := unmarshaledConfig.CreatedAt
+	registryUpdated := registry.UpdatedAt
+	registryCreated := registry.CreatedAt
+
+	// avoid flake where update at seems to be the same (possibly running to fast)
+	time.Sleep(500 * time.Nanosecond)
+
+	// Update the config
+	dbConfig = rt.NewDbConfig()
+	RequireStatus(t, rt.UpsertDbConfig("db1", dbConfig), http.StatusCreated)
+
+	resp = rt.SendAdminRequest(http.MethodGet, "/db1/_config", "")
+	RequireStatus(t, resp, http.StatusOK)
+	unmarshaledConfig = DbConfig{}
+	require.NoError(t, json.Unmarshal(resp.BodyBytes(), &unmarshaledConfig))
+
+	registry = &GatewayRegistry{}
+	_, err = sc.BootstrapContext.Connection.GetMetadataDocument(ctx, b.GetName(), base.SGRegistryKey, registry)
+	require.NoError(t, err)
+
+	// asser that the db config timestamps are as expected
+	assert.Greater(t, unmarshaledConfig.UpdatedAt.UnixNano(), currUpdatedTime.UnixNano())
+	assert.Equal(t, unmarshaledConfig.CreatedAt.UnixNano(), currCreatedTime.UnixNano())
+	// assert that registry timestamps are as expected
+	assert.Equal(t, registry.CreatedAt.UnixNano(), registryCreated.UnixNano())
+	assert.Greater(t, registry.UpdatedAt.UnixNano(), registryUpdated.UnixNano())
+}

rest/config_manager.go

@@ -11,6 +11,7 @@ package rest
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/couchbase/sync_gateway/base"
 	"github.com/couchbase/sync_gateway/db"
@@ -105,6 +106,7 @@ func (b *bootstrapContext) InsertConfig(ctx context.Context, bucketName, groupID
 		}
 
 		// Persist registry
+		registry.UpdatedAt = time.Now().UTC()
 		writeErr := b.setGatewayRegistry(ctx, bucketName, registry)
 		if writeErr == nil {
 			base.DebugfCtx(ctx, base.KeyConfig, "Registry updated successfully")
@@ -131,6 +133,9 @@ func (b *bootstrapContext) InsertConfig(ctx context.Context, bucketName, groupID
 		return 0, fmt.Errorf("InsertConfig failed to persist registry after %d attempts", configUpdateMaxRetryAttempts)
 	}
 	// Step 3. Write the database config
+	timeUpdated := time.Now().UTC()
+	config.UpdatedAt = &timeUpdated
+	config.CreatedAt = &timeUpdated
 	cas, configErr := b.Connection.InsertMetadataDocument(ctx, bucketName, PersistentConfigKey(ctx, groupID, dbName), config)
 	if configErr != nil {
 		base.InfofCtx(ctx, base.KeyConfig, "Insert for database config returned error %v", configErr)
@@ -150,6 +155,7 @@ func (b *bootstrapContext) UpdateConfig(ctx context.Context, bucketName, groupID
 	var updatedConfig *DatabaseConfig
 	var registry *GatewayRegistry
 	var previousVersion string
+	var createdAtTime *time.Time
 
 	registryUpdated := false
 	for attempt := 1; attempt <= configUpdateMaxRetryAttempts; attempt++ {
@@ -167,6 +173,7 @@ func (b *bootstrapContext) UpdateConfig(ctx context.Context, bucketName, groupID
 		if existingConfig == nil {
 			return 0, base.ErrNotFound
 		}
+		createdAtTime = existingConfig.CreatedAt
 
 		base.DebugfCtx(ctx, base.KeyConfig, "UpdateConfig fetched registry and database successfully")
 
@@ -195,6 +202,7 @@ func (b *bootstrapContext) UpdateConfig(ctx context.Context, bucketName, groupID
 		}
 
 		// Persist registry
+		registry.UpdatedAt = time.Now().UTC()
 		writeErr := b.setGatewayRegistry(ctx, bucketName, registry)
 		if writeErr == nil {
 			base.DebugfCtx(ctx, base.KeyConfig, "UpdateConfig persisted updated registry successfully")
@@ -222,6 +230,9 @@ func (b *bootstrapContext) UpdateConfig(ctx context.Context, bucketName, groupID
 	}
 
 	// Step 2. Update the config document
+	timeUpdated := time.Now().UTC()
+	updatedConfig.UpdatedAt = &timeUpdated
+	updatedConfig.CreatedAt = createdAtTime
 	docID := PersistentConfigKey(ctx, groupID, dbName)
 	casOut, err := b.Connection.WriteMetadataDocument(ctx, bucketName, docID, updatedConfig.cfgCas, updatedConfig)
 	if err != nil {

rest/config_registry.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/couchbase/sync_gateway/base"
 )
@@ -47,6 +48,8 @@ type GatewayRegistry struct {
 	Version      string                          `json:"version"`       // Registry version
 	ConfigGroups map[string]*RegistryConfigGroup `json:"config_groups"` // Map of config groups, keyed by config group ID
 	SGVersion    base.ComparableBuildVersion     `json:"sg_version"`    // Latest patch version of Sync Gateway that touched the registry
+	UpdatedAt    time.Time                       `json:"updated_at"`    // Time the registry was last updated
+	CreatedAt    time.Time                       `json:"created_at"`    // Time the registry was created
 }
 
 const GatewayRegistryVersion = "1.0"
@@ -111,6 +114,7 @@ func NewGatewayRegistry(syncGatewayVersion base.ComparableBuildVersion) *Gateway
 		ConfigGroups: make(map[string]*RegistryConfigGroup),
 		Version:      GatewayRegistryVersion,
 		SGVersion:    syncGatewayVersion,
+		CreatedAt:    time.Now().UTC(),
 	}
 }