Move ClickHouse metrics from Counter to Histogram

pinax-network · Jan 9, 2025 · ef3575e · ef3575e
1 parent 691112d
commit ef3575e
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 7 deletions.
diff --git a/src/clickhouse/makeQuery.ts b/src/clickhouse/makeQuery.ts
@@ -14,9 +14,9 @@ export async function makeQuery<T = unknown>(query: string, query_params: ValidQ
 
     prometheus.query.inc();
     if ( data.statistics ) {
-        prometheus.bytes_read.inc(data.statistics.bytes_read);
-        prometheus.rows_read.inc(data.statistics.rows_read);
-        prometheus.elapsed.inc(data.statistics.elapsed);
+        prometheus.bytes_read.observe(data.statistics.bytes_read);
+        prometheus.rows_read.observe(data.statistics.rows_read);
+        prometheus.elapsed.observe(data.statistics.elapsed);
     }
 
     logger.trace({ statistics: data.statistics, rows: data.rows, rows_before_limit_at_least: data.rows_before_limit_at_least });

diff --git a/src/prometheus.ts b/src/prometheus.ts
@@ -1,5 +1,5 @@
 // From https://github.com/pinax-network/substreams-sink-websockets/blob/main/src/prometheus.ts
-import client, { Counter, CounterConfiguration, Gauge, GaugeConfiguration } from 'prom-client';
+import client, { Counter, CounterConfiguration, Gauge, GaugeConfiguration, Histogram, HistogramConfiguration } from 'prom-client';
 import { logger } from "./logger.js";
 
 export const registry = new client.Registry();
@@ -27,6 +27,17 @@ export function registerGauge(name: string, help = "help", labelNames: string[]
     }
 }
 
+export function registerHistogram(name: string, help = "help", labelNames: string[] = [], config?: HistogramConfiguration<string>) {
+    try {
+        registry.registerMetric(new Histogram({ name, help, labelNames, ...config }));
+        logger.debug(`Registered new histogram metric: ${name}`);
+        return registry.getSingleMetric(name) as Histogram;
+    } catch (e) {
+        logger.error("Error registering histogram:", { name, e });
+        throw new Error(`${e}`);
+    }
+}
+
 export async function getSingleMetric(name: string) {
     const metric = registry.getSingleMetric(name);
     const get = await metric?.get();
@@ -37,6 +48,18 @@ export async function getSingleMetric(name: string) {
 export const request_error = registerCounter('request_error', 'Total Requests errors', ['pathname', 'status']);
 export const request = registerCounter('request', 'Total Requests', ['pathname']);
 export const query = registerCounter('query', 'Clickhouse DB queries made');
-export const bytes_read = registerCounter('bytes_read', 'Clickhouse DB Statistics bytes read');
-export const rows_read = registerCounter('rows_read', 'Clickhouse DB Statistics rows read');
-export const elapsed = registerCounter('elapsed', 'Clickhouse DB Statistics query elapsed time');
+export const bytes_read = registerHistogram('bytes_read', 'Clickhouse DB Statistics bytes read', [],
+    { 
+        buckets: Array.from({ length: 63 }, (_, i) => (i % 9 + 1) * 10 ** (6 + Math.floor(i / 9))) // 1Mb to 10Tb buckets, each divided by 10
+    }
+);
+export const rows_read = registerHistogram('rows_read', 'Clickhouse DB Statistics rows read', [],
+    { 
+        buckets: Array.from({ length: 54 }, (_, i) => (i % 9 + 1) * 10 ** (3 + Math.floor(i / 9))) // 1k to 100M, each divided by 10
+    }
+);
+export const elapsed = registerHistogram('elapsed', 'Clickhouse DB Statistics query elapsed time (seconds)', [],
+    { 
+        buckets: [0.001, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30] // In seconds
+    }
+);