diff --git a/config/default.json b/config/default.json index 4ac937c0..6d562fc5 100644 --- a/config/default.json +++ b/config/default.json @@ -80,6 +80,18 @@ "tableName": "knex_migrations" } }, + "replica_db": { + "client": "postgresql", + "connection": { + "database": "@@REPLICA_DB_NAME", + "host": "@@REPLICA_DB_HOST", + "user": "@@REPLICA_DB_USERNAME", + "password": "@@REPLICA_DB_PASSWORD", + "port": "@@REPLICA_DB_PORT", + "connectionString": "" + }, + "debug": false + }, "queue": { "type": "sqs", "awsRegion": "us-east-1", diff --git a/config/env/dev.json b/config/env/dev.json index a59d7b58..f0e9ed76 100644 --- a/config/env/dev.json +++ b/config/env/dev.json @@ -75,6 +75,15 @@ "port": "@@DB_PORT" } }, + "replica_db": { + "connection": { + "database": "@@REPLICA_DB_NAME", + "host": "@@REPLICA_DB_HOST", + "user": "@@REPLICA_DB_USERNAME", + "password": "@@REPLICA_DB_PASSWORD", + "port": "@@REPLICA_DB_PORT" + } + }, "queue": { "type": "sqs", "awsRegion": "@@AWS_REGION", diff --git a/config/env/prod.json b/config/env/prod.json index a59d7b58..f0e9ed76 100644 --- a/config/env/prod.json +++ b/config/env/prod.json @@ -75,6 +75,15 @@ "port": "@@DB_PORT" } }, + "replica_db": { + "connection": { + "database": "@@REPLICA_DB_NAME", + "host": "@@REPLICA_DB_HOST", + "user": "@@REPLICA_DB_USERNAME", + "password": "@@REPLICA_DB_PASSWORD", + "port": "@@REPLICA_DB_PORT" + } + }, "queue": { "type": "sqs", "awsRegion": "@@AWS_REGION", diff --git a/config/env/test.json b/config/env/test.json index 59b3598f..64117e35 100644 --- a/config/env/test.json +++ b/config/env/test.json @@ -57,6 +57,16 @@ "tableName": "knex_migrations" } }, + "replica_db": { + "connection": { + "database": "@@REPLICA_DB_NAME", + "host": "@@REPLICA_DB_HOST", + "user": "@@REPLICA_DB_USERNAME", + "password": "@@REPLICA_DB_PASSWORD", + "port": "@@REPLICA_DB_PORT", + "connectionString": "@@DATABASE_URL" + } + }, "queue": { "type": "sqs", "awsRegion": "us-east-1", diff --git a/package-lock.json b/package-lock.json index 3ef083b6..e4a9a421 100644 --- a/package-lock.json +++ b/package-lock.json @@ -105,7 +105,7 @@ "supertest": "^6.3.3", "tmp-promise": "^3.0.3", "ts-essentials": "^9.3.2", - "typescript": "^5.0.4" + "typescript": "^5.4.5" } }, "node_modules/@ampproject/remapping": { @@ -25899,16 +25899,16 @@ "integrity": "sha512-7uc1O8h1M1g0rArakJdf0uLRSSgFcYexrVoKo+bzJd32gd4gDy2L/Z+8/FjPnU9ydY3pEnVPtr9FyscYY60K1g==" }, "node_modules/typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", + "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", "dev": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" }, "engines": { - "node": ">=12.20" + "node": ">=14.17" } }, "node_modules/u3": { @@ -46618,9 +46618,9 @@ "integrity": "sha512-7uc1O8h1M1g0rArakJdf0uLRSSgFcYexrVoKo+bzJd32gd4gDy2L/Z+8/FjPnU9ydY3pEnVPtr9FyscYY60K1g==" }, "typescript": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", - "integrity": "sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==", + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", + "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", "dev": true }, "u3": { diff --git a/package.json b/package.json index 5ac294ce..2fb0e4e2 100644 --- a/package.json +++ b/package.json @@ -142,7 +142,7 @@ "supertest": "^6.3.3", "tmp-promise": "^3.0.3", "ts-essentials": "^9.3.2", - "typescript": "^5.0.4" + "typescript": "^5.4.5" }, "release-it": { "git": { diff --git a/src/__tests__/ceramic_integration.test.ts b/src/__tests__/ceramic_integration.test.ts index 9cd8218d..7ae87558 100644 --- a/src/__tests__/ceramic_integration.test.ts +++ b/src/__tests__/ceramic_integration.test.ts @@ -25,7 +25,7 @@ import type { GanacheServer } from './make-ganache.util.js' import tmp from 'tmp-promise' import getPort from 'get-port' import type { Knex } from 'knex' -import { clearTables, createDbConnection } from '../db-connection.js' +import { clearTables, createDbConnection, createReplicaDbConnection } from '../db-connection.js' import { CeramicAnchorApp } from '../app.js' import { config } from 'node-config-ts' import cloneDeep from 'lodash.clonedeep' @@ -184,7 +184,8 @@ interface MinimalCASConfig { async function makeCAS( container: Injector, dbConnection: Knex, - minConfig: MinimalCASConfig + minConfig: MinimalCASConfig, + replicaDbConnection: Knex ): Promise { const configCopy = cloneDeep(config) configCopy.mode = minConfig.mode @@ -204,7 +205,7 @@ async function makeCAS( mode: 'inmemory', } return new CeramicAnchorApp( - container.provideValue('config', configCopy).provideValue('dbConnection', dbConnection) + container.provideValue('config', configCopy).provideValue('dbConnection', dbConnection).provideValue('replicaDbConnection', replicaDbConnection) ) } @@ -256,6 +257,9 @@ describe('Ceramic Integration Test', () => { let dbConnection1: Knex let dbConnection2: Knex + let replicaDbConnection1: Knex + let replicaDbConnection2: Knex + let casPort1: number let cas1: CeramicAnchorApp let anchorService1: AnchorService @@ -319,12 +323,15 @@ describe('Ceramic Integration Test', () => { await anchorLauncher.stop() }) + // TODO_WS2-3238_1 : update tests to test with replica db connection as well + // TODO_WS2-3238_2 : make hermetic env have replica db connection describe('Using anchor version 1', () => { beforeAll(async () => { const useSmartContractAnchors = true // Start anchor services dbConnection1 = await createDbConnection() + replicaDbConnection1 = await createReplicaDbConnection() casPort1 = await getPort() cas1 = await makeCAS(createInjector(), dbConnection1, { @@ -333,10 +340,11 @@ describe('Ceramic Integration Test', () => { ganachePort: ganacheServer.port, port: casPort1, useSmartContractAnchors, - }) + }, replicaDbConnection1) await cas1.start() anchorService1 = cas1.container.resolve('anchorService') dbConnection2 = await teeDbConnection(dbConnection1) + replicaDbConnection2 = await createReplicaDbConnection() const casPort2 = await getPort() cas2 = await makeCAS(createInjector(), dbConnection2, { mode: 'server', @@ -344,7 +352,7 @@ describe('Ceramic Integration Test', () => { ganachePort: ganacheServer.port, port: casPort2, useSmartContractAnchors, - }) + }, replicaDbConnection2) await cas2.start() anchorService2 = cas2.container.resolve('anchorService') @@ -368,6 +376,7 @@ describe('Ceramic Integration Test', () => { await cas1.stop() await cas2.stop() await Promise.all([dbConnection1.destroy(), dbConnection2.destroy()]) + await Promise.all([replicaDbConnection1.destroy(), replicaDbConnection2.destroy()]) await Promise.all([ceramic1.close(), ceramic2.close()]) }) @@ -532,6 +541,7 @@ describe('CAR file', () => { const casIPFS = await createIPFS(ipfsApiPort) const ganacheServer = await makeGanache() const dbConnection = await createDbConnection() + const dummyReplicaDbConnection = await createReplicaDbConnection() const casPort = await getPort() const cas = await makeCAS(createInjector(), dbConnection, { mode: 'server', @@ -539,7 +549,7 @@ describe('CAR file', () => { ganachePort: ganacheServer.port, port: casPort, useSmartContractAnchors: true, - }) + }, dummyReplicaDbConnection) await cas.start() const ceramicIPFS = await createIPFS(await getPort()) @@ -608,6 +618,7 @@ describe('Metrics Options', () => { const casIPFS = await createIPFS(ipfsApiPort) const ganacheServer = await makeGanache() const dbConnection = await createDbConnection() + const dummyReplicaDbConnection = await createReplicaDbConnection() const casPort = await getPort() const cas = await makeCAS(createInjector(), dbConnection, { mode: 'server', @@ -618,7 +629,7 @@ describe('Metrics Options', () => { metrics: { instanceIdentifier: '234fffffffffffffffffffffffffffffffff9726129', }, - }) + }, dummyReplicaDbConnection) await cas.start() // Teardown await cas.stop() @@ -632,7 +643,7 @@ describe('Metrics Options', () => { metrics: { instanceIdentifier: '', }, - }) + }, dummyReplicaDbConnection) await cas2.start() await cas2.stop() diff --git a/src/app.ts b/src/app.ts index e4d8856f..8ef6ba91 100644 --- a/src/app.ts +++ b/src/app.ts @@ -14,6 +14,7 @@ import { BlockchainService } from './services/blockchain/blockchain-service.js' import { HTTPEventProducerService } from './services/event-producer/http/http-event-producer-service.js' import { AnchorRepository } from './repositories/anchor-repository.js' import { RequestRepository } from './repositories/request-repository.js' +import { ReplicationRequestRepository } from './repositories/replication-request-repository.js' import { TransactionRepository } from './repositories/transaction-repository.js' import { HealthcheckController } from './controllers/healthcheck-controller.js' import { AnchorController } from './controllers/anchor-controller.js' @@ -48,11 +49,13 @@ import { makeWitnessService, type IWitnessService } from './services/witness-ser type DependenciesContext = { config: Config dbConnection: Knex + replicaDbConnection: Knex } type ProvidedContext = { anchorService: AnchorService requestRepository: RequestRepository + replicationRequestRepository: ReplicationRequestRepository anchorRepository: AnchorRepository transactionRepository: TransactionRepository blockchainService: BlockchainService @@ -98,6 +101,7 @@ export class CeramicAnchorApp { .provideFactory('requestRepository', RequestRepository.make) .provideClass('anchorRepository', AnchorRepository) .provideClass('transactionRepository', TransactionRepository) + .provideClass('replicationRequestRepository', ReplicationRequestRepository) // register services .provideFactory('blockchainService', EthereumBlockchainService.make) .provideClass('eventProducerService', HTTPEventProducerService) @@ -113,8 +117,8 @@ export class CeramicAnchorApp { .provideClass('healthcheckService', HealthcheckService) .provideClass('requestPresentationService', RequestPresentationService) .provideClass('anchorRequestParamsParser', AnchorRequestParamsParser) - .provideClass('requestService', RequestService) .provideClass('continualAnchoringScheduler', TaskSchedulerService) + .provideClass('requestService', RequestService) try { Metrics.start( @@ -130,7 +134,7 @@ export class CeramicAnchorApp { Metrics.count('HELLO', 1) logger.imp('Metrics exporter started') if (this.config.metrics.instanceIdentifier) { - Metrics.setInstanceIdentifier(this.config.metrics.instanceIdentifier) + Metrics.setInstanceIdentifier(this.config.metrics.instanceIdentifier) } } catch (e: any) { logger.imp('ERROR: Metrics exporter failed to start. Continuing anyway.') diff --git a/src/controllers/__tests__/request-controller.test.ts b/src/controllers/__tests__/request-controller.test.ts index 38a5b94f..6e3f1cc5 100644 --- a/src/controllers/__tests__/request-controller.test.ts +++ b/src/controllers/__tests__/request-controller.test.ts @@ -1,5 +1,5 @@ import { describe, expect, jest, test, beforeAll, afterAll } from '@jest/globals' -import { createDbConnection, clearTables } from '../../db-connection.js' +import { createDbConnection, clearTables, createReplicaDbConnection } from '../../db-connection.js' import { createInjector, Injector } from 'typed-inject' import { config } from 'node-config-ts' import { RequestController } from '../request-controller.js' @@ -32,11 +32,13 @@ import { RequestService } from '../../services/request-service.js' import { ValidationSqsQueueService } from '../../services/queue/sqs-queue-service.js' import { makeWitnessService } from '../../services/witness-service.js' import { makeMerkleCarService } from '../../services/merkle-car-service.js' +import { ReplicationRequestRepository } from '../../repositories/replication-request-repository.js' type Tokens = { requestController: RequestController requestRepository: RequestRepository metadataService: IMetadataService + replicationRequestRepository: ReplicationRequestRepository } const FAKE_STREAM_ID_1 = StreamID.fromString( @@ -80,19 +82,24 @@ class MockMetadataService implements IMetadataService { // TODO: CDB-2287 Add tests checking for expected errors when missing/malformed CID/StreamID/GenesisCommit // are detected in a CAR file +// TODO: WS2-3238 Add calls to replica db connection in the test as well describe('createRequest', () => { let dbConnection: Knex + let replicaDbConnection: Knex let container: Injector let controller: RequestController beforeAll(async () => { dbConnection = await createDbConnection() + replicaDbConnection = await createReplicaDbConnection() await clearTables(dbConnection) container = createInjector() .provideValue('config', config) .provideValue('dbConnection', dbConnection) + .provideValue('replicaDbConnection', replicaDbConnection) .provideClass('metadataRepository', MetadataRepository) .provideFactory('requestRepository', RequestRepository.make) + .provideClass('replicationRequestRepository', ReplicationRequestRepository) .provideClass('anchorRepository', AnchorRepository) .provideClass('ipfsService', MockIpfsService) .provideFactory('merkleCarService', makeMerkleCarService) @@ -343,13 +350,19 @@ describe('createRequest', () => { const requestRepository = container.resolve('requestRepository') const findByCidSpy = jest.spyOn(requestRepository, 'findByCid') + const replicaRequestRepository = container.resolve('replicationRequestRepository') + const findByCidSpyReplica = jest.spyOn(replicaRequestRepository, 'findByCid') const res0 = mockResponse() const res1 = mockResponse() await Promise.all([controller.createRequest(req, res0), controller.createRequest(req, res1)]) - - expect(findByCidSpy).toBeCalledTimes(1) - expect(findByCidSpy).toBeCalledWith(cid) + try { + expect(findByCidSpyReplica).toBeCalledTimes(1) + expect(findByCidSpyReplica).toBeCalledWith(cid) + } catch (err) { + expect(findByCidSpy).toBeCalledTimes(1) + expect(findByCidSpy).toBeCalledWith(cid) + } const status0 = res0.status.mock.calls[0][0] const status1 = res1.status.mock.calls[0][0] @@ -370,8 +383,10 @@ describe('createRequest', () => { queue: { sqsQueueUrl: 'testurl' }, }) .provideValue('dbConnection', dbConnection) + .provideValue('replicaDbConnection', replicaDbConnection) .provideClass('metadataRepository', MetadataRepository) .provideFactory('requestRepository', RequestRepository.make) + .provideClass('replicationRequestRepository', ReplicationRequestRepository) .provideClass('anchorRepository', AnchorRepository) .provideClass('ipfsService', MockIpfsService) .provideFactory('merkleCarService', makeMerkleCarService) diff --git a/src/db-connection.ts b/src/db-connection.ts index e8c6a2fa..b993780c 100644 --- a/src/db-connection.ts +++ b/src/db-connection.ts @@ -1,5 +1,5 @@ import { config } from 'node-config-ts' -import type { Db } from 'node-config-ts' +import type { Db, Replicadb } from 'node-config-ts' import type { Knex } from 'knex' import knex from 'knex' import snakeCase from 'lodash.snakecase' @@ -69,6 +69,38 @@ export async function createDbConnection(dbConfig: Db = config.db): Promise { + const replicaKnexConfig: Knex.Config = { + client: replica_db_config.client, + connection: replica_db_config.connection.connectionString || { + host: replica_db_config.connection.host, + port: replica_db_config.connection.port, + user: replica_db_config.connection.user, + password: replica_db_config.connection.password, + database: replica_db_config.connection.database, + }, + debug: replica_db_config.debug, + pool: { min: 3, max: 30 }, + // In our DB, identifiers have snake case formatting while in our code identifiers have camel case formatting. + // We use the following transformers so we can always use camel case formatting in our code. + + // transforms identifier names in our queries from camel case to snake case. This is because the DB uses snake case identifiers. + wrapIdentifier: (value, origWrap): string => origWrap(snakeCase(value)), + // modifies returned rows from the DB. This transforms identifiers from snake case to camel case. + postProcessResponse: (result) => toCamelCase(result), + } + let connection + try { + connection = knex(replicaKnexConfig) + } catch (e) { + throw new Error(`Replica database connection failed: ${e}`) + } + + return connection +} + /** * USED FOR TESTING * Clears all tables diff --git a/src/main.ts b/src/main.ts index 46f6b41e..c91296e0 100644 --- a/src/main.ts +++ b/src/main.ts @@ -3,17 +3,20 @@ import 'reflect-metadata' import { CeramicAnchorApp } from './app.js' import { logger } from './logger/index.js' import { config } from 'node-config-ts' -import { createDbConnection } from './db-connection.js' +import { createDbConnection, createReplicaDbConnection } from './db-connection.js' import { createInjector } from 'typed-inject' async function startApp() { logger.imp('Connecting to database...') const connection = await createDbConnection() logger.imp(`Connected to database: ${config.db.client}`) + const replicaConnection = await createReplicaDbConnection() + logger.imp(`Connected to replica database: ${config.db.client}`) const container = createInjector() - .provideValue('dbConnection', connection) .provideValue('config', config) + .provideValue('dbConnection', connection) + .provideValue('replicaDbConnection', replicaConnection) const app = new CeramicAnchorApp(container) await app.start() diff --git a/src/repositories/replication-request-repository.ts b/src/repositories/replication-request-repository.ts new file mode 100644 index 00000000..9db4424d --- /dev/null +++ b/src/repositories/replication-request-repository.ts @@ -0,0 +1,41 @@ +import { Knex } from 'knex' +import { Request } from '../models/request.js' +import { CID } from 'multiformats/cid' + +const TABLE_NAME = 'request' + +/** + * Replication request repository. + */ +export interface IReplicationRequestRepository { + readonly table: Knex.QueryBuilder + /** + * Finds a request with the given CID if exists using the replica database. + * @param cid CID the request is for + * @returns Promise for the associated request + */ + findByCid(cid: CID | string): Promise +} + +export class ReplicationRequestRepository implements IReplicationRequestRepository { + static inject = ['replicaDbConnection'] as const + + constructor(private readonly connection: Knex) {} + + get table(): Knex.QueryBuilder { + return this.connection(TABLE_NAME) + } + /** + * Finds a request with the given CID if exists using the replica database. + * @param cid CID the request is for + * @returns Promise for the associated request + */ + async findByCid(cid: CID | string): Promise { + const found = await this.table.where({ cid: String(cid) }).first() + if (found) { + return new Request(found) + } + return undefined + } + // Add more methods that utilize the replica connection here +} diff --git a/src/services/request-service.ts b/src/services/request-service.ts index a60f6a7b..ccf824f1 100644 --- a/src/services/request-service.ts +++ b/src/services/request-service.ts @@ -13,6 +13,7 @@ import { IQueueProducerService } from './queue/queue-service.type.js' import { RequestQMessage } from '../models/queue-message.js' import type { OutputOf } from 'codeco' import type { CASResponse } from '@ceramicnetwork/codecs' +import { IReplicationRequestRepository } from '../repositories/replication-request-repository.js' const ISO8601_DATE_FORMAT = new Intl.DateTimeFormat('sv-SE', { month: '2-digit', @@ -32,6 +33,7 @@ export class RequestService { static inject = [ 'config', 'requestRepository', + 'replicationRequestRepository', 'requestPresentationService', 'metadataService', 'validationQueueService', @@ -40,6 +42,7 @@ export class RequestService { constructor( config: Config, private readonly requestRepository: RequestRepository, + private readonly replicationRequestRepository: IReplicationRequestRepository, private readonly requestPresentationService: RequestPresentationService, private readonly metadataService: IMetadataService, private readonly validationQueueService: IQueueProducerService @@ -47,24 +50,44 @@ export class RequestService { this.publishToQueue = Boolean(config.queue.sqsQueueUrl) } + /** + * Finds a request by CID from read replica first if not found there then from main db + * @param cid The CID of the request. + * @returns The request. + */ async getStatusForCid(cid: CID): Promise | { error: string }> { - const request = await this.requestRepository.findByCid(cid) + let request = await this.replicationRequestRepository.findByCid(cid) if (!request) { - throw new RequestDoesNotExistError(cid) + logger.debug(`Request not found in replica db for ${cid}, fetching from main_db`) + Metrics.count(METRIC_NAMES.REPLICA_DB_REQUEST_NOT_FOUND, 1) + request = await this.requestRepository.findByCid(cid) + if (!request) { + throw new RequestDoesNotExistError(cid) + } } - logger.debug( `Found request for ${cid} of ${request.streamId} created at ${ISO8601_DATE_FORMAT.format( request.createdAt )}` ) - return this.requestPresentationService.body(request) } + /** + * Finds a request by CID from read replica first if not found there then from main db + * @param cid The CID of the request. + * @returns The request. + */ async findByCid(cid: CID): Promise | undefined> { - const found = await this.requestRepository.findByCid(cid) - if (!found) return undefined + let found = await this.replicationRequestRepository.findByCid(cid) + if (!found) { + found = await this.requestRepository.findByCid(cid) + logger.debug(`Request not found in replica db for ${cid}, fetching from main_db`) + Metrics.count(METRIC_NAMES.REPLICA_DB_REQUEST_NOT_FOUND, 1) + if (!found) { + throw new RequestDoesNotExistError(cid) + } + } return this.requestPresentationService.body(found) } @@ -109,7 +132,7 @@ export class RequestService { crt: storedRequest.createdAt, org: origin, }) - Metrics.count(METRIC_NAMES.PUBLISH_TO_QUEUE, 1) + Metrics.count(METRIC_NAMES.PUBLISH_TO_QUEUE, 1) } else { await this.requestRepository.markReplaced(storedRequest) Metrics.count(METRIC_NAMES.UPDATED_STORED_REQUEST, 1) @@ -126,10 +149,10 @@ export class RequestService { stream: request.streamId, origin: request.origin, cacao: 'cacaoDomain' in params ? params.cacaoDomain : '', - }; + } // DO NOT REMOVE - this logging is used by business metrics - logger.imp(`Anchor request received: ${JSON.stringify(logData)}`); + logger.imp(`Anchor request received: ${JSON.stringify(logData)}`) return this.requestPresentationService.body(storedRequest) } diff --git a/src/settings.ts b/src/settings.ts index 81cac8fd..616dae50 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -9,7 +9,7 @@ export enum METRIC_NAMES { // Happy path ACCEPTED_REQUESTS = 'accepted_requests', // Anchor service: request candidates accepted - ANCHOR_SUCCESS = 'anchor_success', // Anchor service: requests successfully anchored + ANCHOR_SUCCESS = 'anchor_success', // Anchor service: requests successfully anchored // Anchor Service Errors and warnings ALREADY_ANCHORED_REQUESTS = 'already_anchored_requests', @@ -60,6 +60,7 @@ export enum METRIC_NAMES { // *******************************************************************// // Request Service WRITE_TOTAL_TSDB = 'write_total_tsdb', // note _tsdb implies handles high cardinality + REPLICA_DB_REQUEST_NOT_FOUND = 'replica_db_request_not_found', // DO NOT change TSDB as it is used downstream MERKLE_CAR_CACHE_HIT = 'merkle_car_cache_hit',