diff --git a/CHANGELOG.md b/CHANGELOG.md index 71f671ee..29247ec4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ project adheres to [Semantic Versioning](http://semver.org/). - feat: exposed `registry.registerCollector()` and `registry.collectors()` methods in TypeScript declaration - Added: complete working example of a pushgateway push in `example/pushgateway.js` +- feat: added support for adding labels to default metrics (#374) - Added CHANGELOG reminder ## [12.0.0] - 2020-02-20 diff --git a/README.md b/README.md index ded11045..be1e8a75 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,16 @@ const prefix = 'my_application_'; collectDefaultMetrics({ prefix }); ``` +To apply generic labels to all default metrics, pass an object to the `labels` property (useful if you're working in a clustered environment): + +```js +const client = require('prom-client'); +const collectDefaultMetrics = client.collectDefaultMetrics; +collectDefaultMetrics({ + labels: { NODE_APP_INSTANCE: process.env.NODE_APP_INSTANCE }, +}); +``` + You can get the full list of metrics by inspecting `client.collectDefaultMetrics.metricsList`. diff --git a/index.d.ts b/index.d.ts index 41e0049f..8be5a4a9 100644 --- a/index.d.ts +++ b/index.d.ts @@ -569,6 +569,7 @@ export interface DefaultMetricsCollectorConfiguration { prefix?: string; gcDurationBuckets?: number[]; eventLoopMonitoringPrecision?: number; + labels?: Object; } /** diff --git a/lib/metrics/eventLoopLag.js b/lib/metrics/eventLoopLag.js index 30266f52..dcd41431 100644 --- a/lib/metrics/eventLoopLag.js +++ b/lib/metrics/eventLoopLag.js @@ -23,23 +23,25 @@ const NODEJS_EVENTLOOP_LAG_P50 = 'nodejs_eventloop_lag_p50_seconds'; const NODEJS_EVENTLOOP_LAG_P90 = 'nodejs_eventloop_lag_p90_seconds'; const NODEJS_EVENTLOOP_LAG_P99 = 'nodejs_eventloop_lag_p99_seconds'; -function reportEventloopLag(start, gauge) { +function reportEventloopLag(start, gauge, labels) { const delta = process.hrtime(start); const nanosec = delta[0] * 1e9 + delta[1]; const seconds = nanosec / 1e9; - gauge.set(seconds); + gauge.set(labels, seconds); } module.exports = (registry, config = {}) => { const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); const registers = registry ? [registry] : undefined; let collect; if (!perf_hooks || !perf_hooks.monitorEventLoopDelay) { collect = () => { const start = process.hrtime(); - setImmediate(reportEventloopLag, start, lag); + setImmediate(reportEventloopLag, start, lag, labels); }; } else { const histogram = perf_hooks.monitorEventLoopDelay({ @@ -49,15 +51,15 @@ module.exports = (registry, config = {}) => { collect = () => { const start = process.hrtime(); - setImmediate(reportEventloopLag, start, lag); + setImmediate(reportEventloopLag, start, lag, labels); - lagMin.set(histogram.min / 1e9); - lagMax.set(histogram.max / 1e9); - lagMean.set(histogram.mean / 1e9); - lagStddev.set(histogram.stddev / 1e9); - lagP50.set(histogram.percentile(50) / 1e9); - lagP90.set(histogram.percentile(90) / 1e9); - lagP99.set(histogram.percentile(99) / 1e9); + lagMin.set(labels, histogram.min / 1e9); + lagMax.set(labels, histogram.max / 1e9); + lagMean.set(labels, histogram.mean / 1e9); + lagStddev.set(labels, histogram.stddev / 1e9); + lagP50.set(labels, histogram.percentile(50) / 1e9); + lagP90.set(labels, histogram.percentile(90) / 1e9); + lagP99.set(labels, histogram.percentile(99) / 1e9); }; } @@ -65,6 +67,7 @@ module.exports = (registry, config = {}) => { name: namePrefix + NODEJS_EVENTLOOP_LAG, help: 'Lag of event loop in seconds.', registers, + labelNames, aggregator: 'average', // Use this one metric's `collect` to set all metrics' values. collect, @@ -73,36 +76,43 @@ module.exports = (registry, config = {}) => { name: namePrefix + NODEJS_EVENTLOOP_LAG_MIN, help: 'The minimum recorded event loop delay.', registers, + labelNames, }); const lagMax = new Gauge({ name: namePrefix + NODEJS_EVENTLOOP_LAG_MAX, help: 'The maximum recorded event loop delay.', registers, + labelNames, }); const lagMean = new Gauge({ name: namePrefix + NODEJS_EVENTLOOP_LAG_MEAN, help: 'The mean of the recorded event loop delays.', registers, + labelNames, }); const lagStddev = new Gauge({ name: namePrefix + NODEJS_EVENTLOOP_LAG_STDDEV, help: 'The standard deviation of the recorded event loop delays.', registers, + labelNames, }); const lagP50 = new Gauge({ name: namePrefix + NODEJS_EVENTLOOP_LAG_P50, help: 'The 50th percentile of the recorded event loop delays.', registers, + labelNames, }); const lagP90 = new Gauge({ name: namePrefix + NODEJS_EVENTLOOP_LAG_P90, help: 'The 90th percentile of the recorded event loop delays.', registers, + labelNames, }); const lagP99 = new Gauge({ name: namePrefix + NODEJS_EVENTLOOP_LAG_P99, help: 'The 99th percentile of the recorded event loop delays.', registers, + labelNames, }); }; diff --git a/lib/metrics/gc.js b/lib/metrics/gc.js index 9e8f972f..08b6f454 100644 --- a/lib/metrics/gc.js +++ b/lib/metrics/gc.js @@ -25,6 +25,8 @@ module.exports = (registry, config = {}) => { } const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); const buckets = config.gcDurationBuckets ? config.gcDurationBuckets : DEFAULT_GC_DURATION_BUCKETS; @@ -32,17 +34,19 @@ module.exports = (registry, config = {}) => { name: namePrefix + NODEJS_GC_DURATION_SECONDS, help: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb.', - labelNames: ['kind'], + labelNames: ['kind', ...labelNames], buckets, registers: registry ? [registry] : undefined, }); const obs = new perf_hooks.PerformanceObserver(list => { const entry = list.getEntries()[0]; - const labels = { kind: kinds[entry.kind] }; // Convert duration from milliseconds to seconds - gcHistogram.observe(labels, entry.duration / 1000); + gcHistogram.observe( + Object.assign({ kind: kinds[entry.kind] }, labels), + entry.duration / 1000, + ); }); // We do not expect too many gc events per second, so we do not use buffering diff --git a/lib/metrics/heapSizeAndUsed.js b/lib/metrics/heapSizeAndUsed.js index b01856df..ae8a145d 100644 --- a/lib/metrics/heapSizeAndUsed.js +++ b/lib/metrics/heapSizeAndUsed.js @@ -11,16 +11,18 @@ module.exports = (registry, config = {}) => { if (typeof process.memoryUsage !== 'function') { return; } + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); const registers = registry ? [registry] : undefined; const namePrefix = config.prefix ? config.prefix : ''; const collect = () => { const memUsage = safeMemoryUsage(); if (memUsage) { - heapSizeTotal.set(memUsage.heapTotal); - heapSizeUsed.set(memUsage.heapUsed); + heapSizeTotal.set(labels, memUsage.heapTotal); + heapSizeUsed.set(labels, memUsage.heapUsed); if (memUsage.external !== undefined) { - externalMemUsed.set(memUsage.external); + externalMemUsed.set(labels, memUsage.external); } } }; @@ -29,6 +31,7 @@ module.exports = (registry, config = {}) => { name: namePrefix + NODEJS_HEAP_SIZE_TOTAL, help: 'Process heap size from Node.js in bytes.', registers, + labelNames, // Use this one metric's `collect` to set all metrics' values. collect, }); @@ -36,11 +39,13 @@ module.exports = (registry, config = {}) => { name: namePrefix + NODEJS_HEAP_SIZE_USED, help: 'Process heap size used from Node.js in bytes.', registers, + labelNames, }); const externalMemUsed = new Gauge({ name: namePrefix + NODEJS_EXTERNAL_MEMORY, help: 'Node.js external memory size in bytes.', registers, + labelNames, }); }; diff --git a/lib/metrics/heapSpacesSizeAndUsed.js b/lib/metrics/heapSpacesSizeAndUsed.js index b59221d8..472be21c 100644 --- a/lib/metrics/heapSpacesSizeAndUsed.js +++ b/lib/metrics/heapSpacesSizeAndUsed.js @@ -14,13 +14,16 @@ module.exports = (registry, config = {}) => { const registers = registry ? [registry] : undefined; const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = ['space', ...Object.keys(labels)]; + const gauges = {}; METRICS.forEach(metricType => { gauges[metricType] = new Gauge({ name: namePrefix + NODEJS_HEAP_SIZE[metricType], help: `Process heap space size ${metricType} from Node.js in bytes.`, - labelNames: ['space'], + labelNames, registers, }); }); @@ -33,9 +36,12 @@ module.exports = (registry, config = {}) => { space.space_name.indexOf('_space'), ); - gauges.total.set({ space: spaceName }, space.space_size); - gauges.used.set({ space: spaceName }, space.space_used_size); - gauges.available.set({ space: spaceName }, space.space_available_size); + gauges.total.set({ space: spaceName, ...labels }, space.space_size); + gauges.used.set({ space: spaceName, ...labels }, space.space_used_size); + gauges.available.set( + { space: spaceName, ...labels }, + space.space_available_size, + ); } }; }; diff --git a/lib/metrics/helpers/processMetricsHelpers.js b/lib/metrics/helpers/processMetricsHelpers.js index 50199fa6..bb4e5895 100644 --- a/lib/metrics/helpers/processMetricsHelpers.js +++ b/lib/metrics/helpers/processMetricsHelpers.js @@ -19,10 +19,10 @@ function aggregateByObjectName(list) { return data; } -function updateMetrics(gauge, data) { +function updateMetrics(gauge, data, labels) { gauge.reset(); for (const key in data) { - gauge.set({ type: key }, data[key]); + gauge.set(Object.assign({ type: key }, labels || {}), data[key]); } } diff --git a/lib/metrics/osMemoryHeap.js b/lib/metrics/osMemoryHeap.js index bceca34c..df8c7c53 100644 --- a/lib/metrics/osMemoryHeap.js +++ b/lib/metrics/osMemoryHeap.js @@ -8,17 +8,20 @@ const PROCESS_RESIDENT_MEMORY = 'process_resident_memory_bytes'; function notLinuxVariant(registry, config = {}) { const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); new Gauge({ name: namePrefix + PROCESS_RESIDENT_MEMORY, help: 'Resident memory size in bytes.', registers: registry ? [registry] : undefined, + labelNames, collect() { const memUsage = safeMemoryUsage(); // I don't think the other things returned from `process.memoryUsage()` is relevant to a standard export if (memUsage) { - this.set(memUsage.rss); + this.set(labels, memUsage.rss); } }, }); diff --git a/lib/metrics/processCpuTotal.js b/lib/metrics/processCpuTotal.js index 887aea7b..ef0aa959 100644 --- a/lib/metrics/processCpuTotal.js +++ b/lib/metrics/processCpuTotal.js @@ -8,6 +8,8 @@ const PROCESS_CPU_SECONDS = 'process_cpu_seconds_total'; module.exports = (registry, config = {}) => { const registers = registry ? [registry] : undefined; const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); let lastCpuUsage = process.cpuUsage(); @@ -15,6 +17,7 @@ module.exports = (registry, config = {}) => { name: namePrefix + PROCESS_CPU_USER_SECONDS, help: 'Total user CPU time spent in seconds.', registers, + labelNames, // Use this one metric's `collect` to set all metrics' values. collect() { const cpuUsage = process.cpuUsage(); @@ -24,20 +27,22 @@ module.exports = (registry, config = {}) => { lastCpuUsage = cpuUsage; - cpuUserUsageCounter.inc(userUsageMicros / 1e6); - cpuSystemUsageCounter.inc(systemUsageMicros / 1e6); - cpuUsageCounter.inc((userUsageMicros + systemUsageMicros) / 1e6); + cpuUserUsageCounter.inc(labels, userUsageMicros / 1e6); + cpuSystemUsageCounter.inc(labels, systemUsageMicros / 1e6); + cpuUsageCounter.inc(labels, (userUsageMicros + systemUsageMicros) / 1e6); }, }); const cpuSystemUsageCounter = new Counter({ name: namePrefix + PROCESS_CPU_SYSTEM_SECONDS, help: 'Total system CPU time spent in seconds.', registers, + labelNames, }); const cpuUsageCounter = new Counter({ name: namePrefix + PROCESS_CPU_SECONDS, help: 'Total user and system CPU time spent in seconds.', registers, + labelNames, }); }; diff --git a/lib/metrics/processHandles.js b/lib/metrics/processHandles.js index 14ccacfb..7ba58738 100644 --- a/lib/metrics/processHandles.js +++ b/lib/metrics/processHandles.js @@ -15,25 +15,28 @@ module.exports = (registry, config = {}) => { const registers = registry ? [registry] : undefined; const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); new Gauge({ name: namePrefix + NODEJS_ACTIVE_HANDLES, help: 'Number of active libuv handles grouped by handle type. Every handle type is C++ class name.', - labelNames: ['type'], + labelNames: ['type', ...labelNames], registers, collect() { const handles = process._getActiveHandles(); - updateMetrics(this, aggregateByObjectName(handles)); + updateMetrics(this, aggregateByObjectName(handles), labels); }, }); new Gauge({ name: namePrefix + NODEJS_ACTIVE_HANDLES_TOTAL, help: 'Total number of active handles.', registers, + labelNames, collect() { const handles = process._getActiveHandles(); - this.set(handles.length); + this.set(labels, handles.length); }, }); }; diff --git a/lib/metrics/processMaxFileDescriptors.js b/lib/metrics/processMaxFileDescriptors.js index 7f717381..f71a85a7 100644 --- a/lib/metrics/processMaxFileDescriptors.js +++ b/lib/metrics/processMaxFileDescriptors.js @@ -28,13 +28,16 @@ module.exports = (registry, config = {}) => { if (maxFds === undefined) return; const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); new Gauge({ name: namePrefix + PROCESS_MAX_FDS, help: 'Maximum number of open file descriptors.', registers: registry ? [registry] : undefined, + labelNames, collect() { - if (maxFds !== undefined) this.set(maxFds); + if (maxFds !== undefined) this.set(labels, maxFds); }, }); }; diff --git a/lib/metrics/processOpenFileDescriptors.js b/lib/metrics/processOpenFileDescriptors.js index dc1ca8ba..547a8e3a 100644 --- a/lib/metrics/processOpenFileDescriptors.js +++ b/lib/metrics/processOpenFileDescriptors.js @@ -12,17 +12,20 @@ module.exports = (registry, config = {}) => { } const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); new Gauge({ name: namePrefix + PROCESS_OPEN_FDS, help: 'Number of open file descriptors.', registers: registry ? [registry] : undefined, + labelNames, collect() { try { const fds = fs.readdirSync('/proc/self/fd'); // Minus 1 to not count the fd that was used by readdirSync(), // it's now closed. - this.set(fds.length - 1); + this.set(labels, fds.length - 1); } catch { // noop } diff --git a/lib/metrics/processRequests.js b/lib/metrics/processRequests.js index 7bc8519c..cb47ee38 100644 --- a/lib/metrics/processRequests.js +++ b/lib/metrics/processRequests.js @@ -13,16 +13,18 @@ module.exports = (registry, config = {}) => { } const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); new Gauge({ name: namePrefix + NODEJS_ACTIVE_REQUESTS, help: 'Number of active libuv requests grouped by request type. Every request type is C++ class name.', - labelNames: ['type'], + labelNames: ['type', ...labelNames], registers: registry ? [registry] : undefined, collect() { const requests = process._getActiveRequests(); - updateMetrics(this, aggregateByObjectName(requests)); + updateMetrics(this, aggregateByObjectName(requests), labels); }, }); @@ -30,9 +32,10 @@ module.exports = (registry, config = {}) => { name: namePrefix + NODEJS_ACTIVE_REQUESTS_TOTAL, help: 'Total number of active requests.', registers: registry ? [registry] : undefined, + labelNames, collect() { const requests = process._getActiveRequests(); - this.set(requests.length); + this.set(labels, requests.length); }, }); }; diff --git a/lib/metrics/processStartTime.js b/lib/metrics/processStartTime.js index cc3e69af..bc41c339 100644 --- a/lib/metrics/processStartTime.js +++ b/lib/metrics/processStartTime.js @@ -7,14 +7,17 @@ const PROCESS_START_TIME = 'process_start_time_seconds'; module.exports = (registry, config = {}) => { const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); new Gauge({ name: namePrefix + PROCESS_START_TIME, help: 'Start time of the process since unix epoch in seconds.', registers: registry ? [registry] : undefined, + labelNames, aggregator: 'omit', collect() { - this.set(startInSeconds); + this.set(labels, startInSeconds); }, }); }; diff --git a/lib/metrics/version.js b/lib/metrics/version.js index 6543dfe1..9cf3fa04 100644 --- a/lib/metrics/version.js +++ b/lib/metrics/version.js @@ -8,11 +8,13 @@ const NODE_VERSION_INFO = 'nodejs_version_info'; module.exports = (registry, config = {}) => { const namePrefix = config.prefix ? config.prefix : ''; + const labels = config.labels ? config.labels : {}; + const labelNames = Object.keys(labels); new Gauge({ name: namePrefix + NODE_VERSION_INFO, help: 'Node.js version info.', - labelNames: ['version', 'major', 'minor', 'patch'], + labelNames: ['version', 'major', 'minor', 'patch', ...labelNames], registers: registry ? [registry] : undefined, aggregator: 'first', collect() { @@ -22,6 +24,7 @@ module.exports = (registry, config = {}) => { versionSegments[0], versionSegments[1], versionSegments[2], + ...Object.values(labels), ).set(1); }, }); diff --git a/test/defaultMetricsTest.js b/test/defaultMetricsTest.js index d59f1d8f..e4330d80 100644 --- a/test/defaultMetricsTest.js +++ b/test/defaultMetricsTest.js @@ -69,6 +69,30 @@ describe('collectDefaultMetrics', () => { } }); + it('should apply labels to metrics when configured', async () => { + expect(await register.getMetricsAsJSON()).toHaveLength(0); + + const labels = { NODE_APP_INSTANCE: 0 }; + collectDefaultMetrics({ labels }); + + const metrics = await register.getMetricsAsJSON(); + + // flatten metric values into a single array + const allMetricValues = metrics.reduce( + (previous, metric) => previous.concat(metric.values), + [], + ); + + // this varies between 45 and 47 depending on node handles - we just wanna + // assert there's at least one so we know the assertions in the loop below + // are executed + expect(allMetricValues.length).toBeGreaterThan(0); + + allMetricValues.forEach(metricValue => { + expect(metricValue.labels).toMatchObject(labels); + }); + }); + describe('disabling', () => { it('should not throw error', () => { const fn = function () {