feat: added support for adding labels to default metrics (#374)

CHANGELOG.md

@@ -49,6 +49,7 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 - feat: exposed `registry.registerCollector()` and `registry.collectors()` methods in TypeScript declaration
 - Added: complete working example of a pushgateway push in `example/pushgateway.js`
+- feat: added support for adding labels to default metrics (#374)
 - Added CHANGELOG reminder
 
 ## [12.0.0] - 2020-02-20

README.md

@@ -80,6 +80,16 @@ const prefix = 'my_application_';
 collectDefaultMetrics({ prefix });
 ```
 
+To apply generic labels to all default metrics, pass an object to the `labels` property (useful if you're working in a clustered environment):
+
+```js
+const client = require('prom-client');
+const collectDefaultMetrics = client.collectDefaultMetrics;
+collectDefaultMetrics({
+  labels: { NODE_APP_INSTANCE: process.env.NODE_APP_INSTANCE },
+});
+```
+
 You can get the full list of metrics by inspecting
 `client.collectDefaultMetrics.metricsList`.
 

index.d.ts

@@ -569,6 +569,7 @@ export interface DefaultMetricsCollectorConfiguration {
 	prefix?: string;
 	gcDurationBuckets?: number[];
 	eventLoopMonitoringPrecision?: number;
+	labels?: Object;
 }
 
 /**

lib/metrics/eventLoopLag.js

@@ -23,23 +23,25 @@ const NODEJS_EVENTLOOP_LAG_P50 = 'nodejs_eventloop_lag_p50_seconds';
 const NODEJS_EVENTLOOP_LAG_P90 = 'nodejs_eventloop_lag_p90_seconds';
 const NODEJS_EVENTLOOP_LAG_P99 = 'nodejs_eventloop_lag_p99_seconds';
 
-function reportEventloopLag(start, gauge) {
+function reportEventloopLag(start, gauge, labels) {
 	const delta = process.hrtime(start);
 	const nanosec = delta[0] * 1e9 + delta[1];
 	const seconds = nanosec / 1e9;
 
-	gauge.set(seconds);
+	gauge.set(labels, seconds);
 }
 
 module.exports = (registry, config = {}) => {
 	const namePrefix = config.prefix ? config.prefix : '';
+	const labels = config.labels ? config.labels : {};
+	const labelNames = Object.keys(labels);
 	const registers = registry ? [registry] : undefined;
 
 	let collect;
 	if (!perf_hooks || !perf_hooks.monitorEventLoopDelay) {
 		collect = () => {
 			const start = process.hrtime();
-			setImmediate(reportEventloopLag, start, lag);
+			setImmediate(reportEventloopLag, start, lag, labels);
 		};
 	} else {
 		const histogram = perf_hooks.monitorEventLoopDelay({
@@ -49,22 +51,23 @@ module.exports = (registry, config = {}) => {
 
 		collect = () => {
 			const start = process.hrtime();
-			setImmediate(reportEventloopLag, start, lag);
+			setImmediate(reportEventloopLag, start, lag, labels);
 
-			lagMin.set(histogram.min / 1e9);
-			lagMax.set(histogram.max / 1e9);
-			lagMean.set(histogram.mean / 1e9);
-			lagStddev.set(histogram.stddev / 1e9);
-			lagP50.set(histogram.percentile(50) / 1e9);
-			lagP90.set(histogram.percentile(90) / 1e9);
-			lagP99.set(histogram.percentile(99) / 1e9);
+			lagMin.set(labels, histogram.min / 1e9);
+			lagMax.set(labels, histogram.max / 1e9);
+			lagMean.set(labels, histogram.mean / 1e9);
+			lagStddev.set(labels, histogram.stddev / 1e9);
+			lagP50.set(labels, histogram.percentile(50) / 1e9);
+			lagP90.set(labels, histogram.percentile(90) / 1e9);
+			lagP99.set(labels, histogram.percentile(99) / 1e9);
 		};
 	}
 
 	const lag = new Gauge({
 		name: namePrefix + NODEJS_EVENTLOOP_LAG,
 		help: 'Lag of event loop in seconds.',
 		registers,
+		labelNames,
 		aggregator: 'average',
 		// Use this one metric's `collect` to set all metrics' values.
 		collect,
@@ -73,36 +76,43 @@ module.exports = (registry, config = {}) => {
 		name: namePrefix + NODEJS_EVENTLOOP_LAG_MIN,
 		help: 'The minimum recorded event loop delay.',
 		registers,
+		labelNames,
 	});
 	const lagMax = new Gauge({
 		name: namePrefix + NODEJS_EVENTLOOP_LAG_MAX,
 		help: 'The maximum recorded event loop delay.',
 		registers,
+		labelNames,
 	});
 	const lagMean = new Gauge({
 		name: namePrefix + NODEJS_EVENTLOOP_LAG_MEAN,
 		help: 'The mean of the recorded event loop delays.',
 		registers,
+		labelNames,
 	});
 	const lagStddev = new Gauge({
 		name: namePrefix + NODEJS_EVENTLOOP_LAG_STDDEV,
 		help: 'The standard deviation of the recorded event loop delays.',
 		registers,
+		labelNames,
 	});
 	const lagP50 = new Gauge({
 		name: namePrefix + NODEJS_EVENTLOOP_LAG_P50,
 		help: 'The 50th percentile of the recorded event loop delays.',
 		registers,
+		labelNames,
 	});
 	const lagP90 = new Gauge({
 		name: namePrefix + NODEJS_EVENTLOOP_LAG_P90,
 		help: 'The 90th percentile of the recorded event loop delays.',
 		registers,
+		labelNames,
 	});
 	const lagP99 = new Gauge({
 		name: namePrefix + NODEJS_EVENTLOOP_LAG_P99,
 		help: 'The 99th percentile of the recorded event loop delays.',
 		registers,
+		labelNames,
 	});
 };
 

lib/metrics/gc.js

@@ -25,24 +25,28 @@ module.exports = (registry, config = {}) => {
 	}
 
 	const namePrefix = config.prefix ? config.prefix : '';
+	const labels = config.labels ? config.labels : {};
+	const labelNames = Object.keys(labels);
 	const buckets = config.gcDurationBuckets
 		? config.gcDurationBuckets
 		: DEFAULT_GC_DURATION_BUCKETS;
 	const gcHistogram = new Histogram({
 		name: namePrefix + NODEJS_GC_DURATION_SECONDS,
 		help:
 			'Garbage collection duration by kind, one of major, minor, incremental or weakcb.',
-		labelNames: ['kind'],
+		labelNames: ['kind', ...labelNames],
 		buckets,
 		registers: registry ? [registry] : undefined,
 	});
 
 	const obs = new perf_hooks.PerformanceObserver(list => {
 		const entry = list.getEntries()[0];
-		const labels = { kind: kinds[entry.kind] };
 
 		// Convert duration from milliseconds to seconds
-		gcHistogram.observe(labels, entry.duration / 1000);
+		gcHistogram.observe(
+			Object.assign({ kind: kinds[entry.kind] }, labels),
+			entry.duration / 1000,
+		);
 	});
 
 	// We do not expect too many gc events per second, so we do not use buffering

lib/metrics/heapSizeAndUsed.js

@@ -11,16 +11,18 @@ module.exports = (registry, config = {}) => {
 	if (typeof process.memoryUsage !== 'function') {
 		return;
 	}
+	const labels = config.labels ? config.labels : {};
+	const labelNames = Object.keys(labels);
 
 	const registers = registry ? [registry] : undefined;
 	const namePrefix = config.prefix ? config.prefix : '';
 	const collect = () => {
 		const memUsage = safeMemoryUsage();
 		if (memUsage) {
-			heapSizeTotal.set(memUsage.heapTotal);
-			heapSizeUsed.set(memUsage.heapUsed);
+			heapSizeTotal.set(labels, memUsage.heapTotal);
+			heapSizeUsed.set(labels, memUsage.heapUsed);
 			if (memUsage.external !== undefined) {
-				externalMemUsed.set(memUsage.external);
+				externalMemUsed.set(labels, memUsage.external);
 			}
 		}
 	};
@@ -29,18 +31,21 @@ module.exports = (registry, config = {}) => {
 		name: namePrefix + NODEJS_HEAP_SIZE_TOTAL,
 		help: 'Process heap size from Node.js in bytes.',
 		registers,
+		labelNames,
 		// Use this one metric's `collect` to set all metrics' values.
 		collect,
 	});
 	const heapSizeUsed = new Gauge({
 		name: namePrefix + NODEJS_HEAP_SIZE_USED,
 		help: 'Process heap size used from Node.js in bytes.',
 		registers,
+		labelNames,
 	});
 	const externalMemUsed = new Gauge({
 		name: namePrefix + NODEJS_EXTERNAL_MEMORY,
 		help: 'Node.js external memory size in bytes.',
 		registers,
+		labelNames,
 	});
 };
 

lib/metrics/heapSpacesSizeAndUsed.js

@@ -14,13 +14,16 @@ module.exports = (registry, config = {}) => {
 	const registers = registry ? [registry] : undefined;
 	const namePrefix = config.prefix ? config.prefix : '';
 
+	const labels = config.labels ? config.labels : {};
+	const labelNames = ['space', ...Object.keys(labels)];
+
 	const gauges = {};
 
 	METRICS.forEach(metricType => {
 		gauges[metricType] = new Gauge({
 			name: namePrefix + NODEJS_HEAP_SIZE[metricType],
 			help: `Process heap space size ${metricType} from Node.js in bytes.`,
-			labelNames: ['space'],
+			labelNames,
 			registers,
 		});
 	});
@@ -33,9 +36,12 @@ module.exports = (registry, config = {}) => {
 				space.space_name.indexOf('_space'),
 			);
 
-			gauges.total.set({ space: spaceName }, space.space_size);
-			gauges.used.set({ space: spaceName }, space.space_used_size);
-			gauges.available.set({ space: spaceName }, space.space_available_size);
+			gauges.total.set({ space: spaceName, ...labels }, space.space_size);
+			gauges.used.set({ space: spaceName, ...labels }, space.space_used_size);
+			gauges.available.set(
+				{ space: spaceName, ...labels },
+				space.space_available_size,
+			);
 		}
 	};
 };

lib/metrics/helpers/processMetricsHelpers.js

@@ -19,10 +19,10 @@ function aggregateByObjectName(list) {
 	return data;
 }
 
-function updateMetrics(gauge, data) {
+function updateMetrics(gauge, data, labels) {
 	gauge.reset();
 	for (const key in data) {
-		gauge.set({ type: key }, data[key]);
+		gauge.set(Object.assign({ type: key }, labels || {}), data[key]);
 	}
 }
 

lib/metrics/osMemoryHeap.js

@@ -8,17 +8,20 @@ const PROCESS_RESIDENT_MEMORY = 'process_resident_memory_bytes';
 
 function notLinuxVariant(registry, config = {}) {
 	const namePrefix = config.prefix ? config.prefix : '';
+	const labels = config.labels ? config.labels : {};
+	const labelNames = Object.keys(labels);
 
 	new Gauge({
 		name: namePrefix + PROCESS_RESIDENT_MEMORY,
 		help: 'Resident memory size in bytes.',
 		registers: registry ? [registry] : undefined,
+		labelNames,
 		collect() {
 			const memUsage = safeMemoryUsage();
 
 			// I don't think the other things returned from `process.memoryUsage()` is relevant to a standard export
 			if (memUsage) {
-				this.set(memUsage.rss);
+				this.set(labels, memUsage.rss);
 			}
 		},
 	});

lib/metrics/processCpuTotal.js

@@ -8,13 +8,16 @@ const PROCESS_CPU_SECONDS = 'process_cpu_seconds_total';
 module.exports = (registry, config = {}) => {
 	const registers = registry ? [registry] : undefined;
 	const namePrefix = config.prefix ? config.prefix : '';
+	const labels = config.labels ? config.labels : {};
+	const labelNames = Object.keys(labels);
 
 	let lastCpuUsage = process.cpuUsage();
 
 	const cpuUserUsageCounter = new Counter({
 		name: namePrefix + PROCESS_CPU_USER_SECONDS,
 		help: 'Total user CPU time spent in seconds.',
 		registers,
+		labelNames,
 		// Use this one metric's `collect` to set all metrics' values.
 		collect() {
 			const cpuUsage = process.cpuUsage();
@@ -24,20 +27,22 @@ module.exports = (registry, config = {}) => {
 
 			lastCpuUsage = cpuUsage;
 
-			cpuUserUsageCounter.inc(userUsageMicros / 1e6);
-			cpuSystemUsageCounter.inc(systemUsageMicros / 1e6);
-			cpuUsageCounter.inc((userUsageMicros + systemUsageMicros) / 1e6);
+			cpuUserUsageCounter.inc(labels, userUsageMicros / 1e6);
+			cpuSystemUsageCounter.inc(labels, systemUsageMicros / 1e6);
+			cpuUsageCounter.inc(labels, (userUsageMicros + systemUsageMicros) / 1e6);
 		},
 	});
 	const cpuSystemUsageCounter = new Counter({
 		name: namePrefix + PROCESS_CPU_SYSTEM_SECONDS,
 		help: 'Total system CPU time spent in seconds.',
 		registers,
+		labelNames,
 	});
 	const cpuUsageCounter = new Counter({
 		name: namePrefix + PROCESS_CPU_SECONDS,
 		help: 'Total user and system CPU time spent in seconds.',
 		registers,
+		labelNames,
 	});
 };
 

lib/metrics/processHandles.js

@@ -15,25 +15,28 @@ module.exports = (registry, config = {}) => {
 
 	const registers = registry ? [registry] : undefined;
 	const namePrefix = config.prefix ? config.prefix : '';
+	const labels = config.labels ? config.labels : {};
+	const labelNames = Object.keys(labels);
 
 	new Gauge({
 		name: namePrefix + NODEJS_ACTIVE_HANDLES,
 		help:
 			'Number of active libuv handles grouped by handle type. Every handle type is C++ class name.',
-		labelNames: ['type'],
+		labelNames: ['type', ...labelNames],
 		registers,
 		collect() {
 			const handles = process._getActiveHandles();
-			updateMetrics(this, aggregateByObjectName(handles));
+			updateMetrics(this, aggregateByObjectName(handles), labels);
 		},
 	});
 	new Gauge({
 		name: namePrefix + NODEJS_ACTIVE_HANDLES_TOTAL,
 		help: 'Total number of active handles.',
 		registers,
+		labelNames,
 		collect() {
 			const handles = process._getActiveHandles();
-			this.set(handles.length);
+			this.set(labels, handles.length);
 		},
 	});
 };

lib/metrics/processMaxFileDescriptors.js

@@ -28,13 +28,16 @@ module.exports = (registry, config = {}) => {
 	if (maxFds === undefined) return;
 
 	const namePrefix = config.prefix ? config.prefix : '';
+	const labels = config.labels ? config.labels : {};
+	const labelNames = Object.keys(labels);
 
 	new Gauge({
 		name: namePrefix + PROCESS_MAX_FDS,
 		help: 'Maximum number of open file descriptors.',
 		registers: registry ? [registry] : undefined,
+		labelNames,
 		collect() {
-			if (maxFds !== undefined) this.set(maxFds);
+			if (maxFds !== undefined) this.set(labels, maxFds);
 		},
 	});
 };

lib/metrics/processOpenFileDescriptors.js

@@ -12,17 +12,20 @@ module.exports = (registry, config = {}) => {
 	}
 
 	const namePrefix = config.prefix ? config.prefix : '';
+	const labels = config.labels ? config.labels : {};
+	const labelNames = Object.keys(labels);
 
 	new Gauge({
 		name: namePrefix + PROCESS_OPEN_FDS,
 		help: 'Number of open file descriptors.',
 		registers: registry ? [registry] : undefined,
+		labelNames,
 		collect() {
 			try {
 				const fds = fs.readdirSync('/proc/self/fd');
 				// Minus 1 to not count the fd that was used by readdirSync(),
 				// it's now closed.
-				this.set(fds.length - 1);
+				this.set(labels, fds.length - 1);
 			} catch {
 				// noop
 			}