From 9046a369e3733cbc1236ff849d457fd1e06ef42f Mon Sep 17 00:00:00 2001 From: Coldwings Date: Fri, 3 Nov 2023 15:24:07 +0800 Subject: [PATCH] fix the metrics exporter Signed-off-by: Coldwings --- src/exporter_handler.h | 80 +++++++++------ src/exporter_server.h | 100 +++++++++---------- src/image_service.cpp | 6 +- src/metrics.h | 222 ----------------------------------------- src/metrics_fs.h | 73 ++++++++------ src/textexporter.h | 16 +++ 6 files changed, 163 insertions(+), 334 deletions(-) delete mode 100644 src/metrics.h diff --git a/src/exporter_handler.h b/src/exporter_handler.h index ddc84333..18786167 100644 --- a/src/exporter_handler.h +++ b/src/exporter_handler.h @@ -1,12 +1,27 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + #pragma once -#include "metrics.h" #include +#include +#include #include -#include -#include "textexporter.h" -#include +#include "textexporter.h" namespace ExposeMetrics { @@ -23,51 +38,58 @@ namespace ExposeMetrics { ret.append(name.help_str()).append("\n"); \ ret.append(name.type_str()).append("\n"); \ for (auto x : va_##name) { \ - ret.append( \ - name.render(x.second->val(), nodename.c_str(), x.first)) \ - .append("\n"); \ + ret.append(name.render(x.second->val(), x.first)).append("\n"); \ } \ ret.append("\n"); \ } - struct ExposeRender : public photon::net::http::HTTPHandler { - EXPOSE_PHOTON_METRICLIST(latency, Metric::AverageLatencyCounter); - - std::string nodename; - photon::net::http::DelegateHTTPHandler handler; + EXPOSE_PHOTON_METRICLIST(throughput, Metric::QPSCounter); + EXPOSE_PHOTON_METRICLIST(qps, Metric::QPSCounter); + EXPOSE_PHOTON_METRICLIST(latency, Metric::MaxLatencyCounter); + EXPOSE_PHOTON_METRICLIST(count, Metric::AddCounter); + EXPOSE_PHOTON_METRICLIST(cache, Metric::ValueCounter); template - ExposeRender(Args&&... args) - : nodename(std::forward(args)...), - handler{this, &ExposeRender::handle_request} {} + ExposeRender(Args&&... args) {} std::string render() { - EXPOSE_TEMPLATE(alive, is_alive : gauge{node}); - EXPOSE_TEMPLATE(latency, blob_read_average_latency : gauge{node, type} #us); + EXPOSE_TEMPLATE(alive, OverlayBD_Alive : gauge{node}); + EXPOSE_TEMPLATE(throughput, OverlayBD_Read_Throughtput + : gauge{node, type, mode} #Bytes / sec); + EXPOSE_TEMPLATE(qps, OverlayBD_QPS : gauge{node, type, mode}); + EXPOSE_TEMPLATE(latency, OverlayBD_MaxLatency + : gauge{node, type, mode} #us); + EXPOSE_TEMPLATE(count, OverlayBD_Count : gauge{node, type} #Bytes); std::string ret(alive.help_str()); ret.append("\n") .append(alive.type_str()) .append("\n") - .append(alive.render(1, nodename.c_str())) + .append(alive.render(1)) .append("\n\n"); + LOOP_APPEND_METRIC(ret, throughput); + LOOP_APPEND_METRIC(ret, qps); LOOP_APPEND_METRIC(ret, latency); + LOOP_APPEND_METRIC(ret, count); return ret; } - int handle_request(photon::net::http::Request& req, photon::net::http::Response& resp, std::string_view) override { - std::string str = render(); - auto cl = str.size(); - if (cl > 4096) { - LOG_ERROR_RETURN(0, -1, "RetType failed test"); - } + int handle_request(photon::net::http::Request& req, + photon::net::http::Response& resp, + std::string_view) override { + auto body = render(); resp.set_result(200); - resp.headers.content_length(cl); - resp.write((void*)str.data(), str.size()); - return 0; + resp.keep_alive(true); + resp.headers.insert("Content-Type", "text/plain; version=0.0.4"); + resp.headers.content_length(body.length()); + ssize_t len = 0; + len = resp.write((void*)body.data(), body.length()); + if (len == (ssize_t)body.length()) { + return 0; + } else { + LOG_ERRNO_RETURN(0, -1, "Failed to write exporter response"); + } } - - photon::net::http::DelegateHTTPHandler get_handler() { return handler; } }; #undef LOOP_APPEND_METRIC diff --git a/src/exporter_server.h b/src/exporter_server.h index e485ad6c..80d0b6e1 100644 --- a/src/exporter_server.h +++ b/src/exporter_server.h @@ -1,81 +1,75 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + #pragma once -#include "config.h" -#include "exporter_handler.h" -#include "metrics_fs.h" +#include #include #include -#include #include #include -#include -#include -#include - -#include - +#include "config.h" +#include "exporter_handler.h" +#include "metrics_fs.h" class OverlayBDMetric { public: - MetricMeta download; - ExposeMetrics::ExposeRender exporter; + MetricMeta pread, download; - photon::Timer timer; - uint64_t m_timeout = -1; + ExposeMetrics::ExposeRender exporter; - OverlayBDMetric() : timer(-1, {this, &OverlayBDMetric::on_timer}) { + OverlayBDMetric() { + exporter.add_throughput("pread", pread.throughput); + exporter.add_latency("pread", pread.latency); + exporter.add_qps("pread", pread.qps); + exporter.add_count("pread", pread.total); + exporter.add_throughput("download", download.throughput); exporter.add_latency("download", download.latency); + exporter.add_qps("download", download.qps); + exporter.add_count("download", download.total); } - - ~OverlayBDMetric() { - timer.cancel(); - timer.stop(); - } - - uint64_t on_timer() { - return m_timeout; - } - - uint64_t interval(uint64_t x) { return m_timeout = x; } - uint64_t interval() { return m_timeout; } - int start() { return timer.reset(0); } }; struct ExporterServer { - photon::WorkPool wp; - photon::net::http::HTTPServer *httpserver = nullptr; photon::net::ISocketServer *tcpserver = nullptr; bool ready = false; ExporterServer(ImageConfigNS::GlobalConfig &config, - OverlayBDMetric *metrics) - : wp(1, photon::INIT_EVENT_EPOLL, 0, 0) { - wp.call([&] { - prctl(PR_SET_THP_DISABLE, 1); - pthread_setname_np(pthread_self(), "overlaybd-exporter"); - char buffer[64]; - snprintf(buffer, 63, "localhost:%d", config.exporterConfig().port()); - metrics->exporter.nodename = buffer; - - tcpserver = photon::net::new_tcp_socket_server(); - tcpserver->bind(config.exporterConfig().port(), photon::net::IPAddr("127.0.0.1")); - - tcpserver->listen(); - httpserver = photon::net::http::new_http_server(); - httpserver->add_handler(&metrics->exporter); - tcpserver->set_handler(httpserver->get_connection_handler()); - tcpserver->start_loop(); - ready = true; - }); + OverlayBDMetric *metrics) { + tcpserver = photon::net::new_tcp_socket_server(); + tcpserver->setsockopt(SOL_SOCKET, SO_REUSEPORT, 1); + if (tcpserver->bind(config.exporterConfig().port()) < 0) + LOG_ERRNO_RETURN(0, , "Failed to bind exporter port `", + config.exporterConfig().port()); + if (tcpserver->listen() < 0) + LOG_ERRNO_RETURN(0, , "Failed to listen exporter port `", + config.exporterConfig().port()); + httpserver = photon::net::http::new_http_server(); + httpserver->add_handler(&metrics->exporter, false, + config.exporterConfig().uriPrefix()); + tcpserver->set_handler(httpserver->get_connection_handler()); + tcpserver->start_loop(); + ready = true; } ~ExporterServer() { - wp.call([&] { - delete tcpserver; - delete httpserver; - }); + delete tcpserver; + delete httpserver; } }; \ No newline at end of file diff --git a/src/image_service.cpp b/src/image_service.cpp index 87048dc9..0075b137 100644 --- a/src/image_service.cpp +++ b/src/image_service.cpp @@ -346,8 +346,6 @@ int ImageService::init() { } if (global_conf.exporterConfig().enable()) { metrics.reset(new OverlayBDMetric()); - metrics->interval(global_conf.exporterConfig().updateInterval()); - metrics->start(); global_fs.srcfs = new MetricFS(global_fs.underlay_registryfs, &metrics->download); exporter = new ExporterServer(global_conf, metrics.get()); if (!exporter->ready) @@ -410,6 +408,10 @@ int ImageService::init() { LOG_ERRNO_RETURN(0, -1, "failed to create cached_fs"); } + if (global_conf.exporterConfig().enable()) { + global_fs.cached_fs = new MetricFS(global_fs.cached_fs, &metrics->pread); + } + if (global_conf.gzipCacheConfig().enable()) { LOG_INFO("use gzip file cache"); cache_dir = global_conf.gzipCacheConfig().cacheDir(); diff --git a/src/metrics.h b/src/metrics.h deleted file mode 100644 index e6f0e664..00000000 --- a/src/metrics.h +++ /dev/null @@ -1,222 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace Metric { - -class ValueCounter { -public: - int64_t counter = 0; - - void set(int64_t x) { counter = x; } - int64_t val() { return counter; } -}; - -class AddCounter { -public: - int64_t counter = 0; - - void inc() { counter++; } - void dec() { counter--; } - void add(int64_t x) { counter += x; } - void sub(int64_t x) { counter -= x; } - void reset() { counter = 0; } - int64_t val() { return counter; } -}; - -class AverageCounter { -public: - int64_t sum = 0; - int64_t cnt = 0; - uint64_t time = 0; - uint64_t m_interval = 60UL * 1000 * 1000; - - void normalize() { - auto now = photon::now; - if (now - time > m_interval * 2) { - reset(); - } else if (now - time > m_interval) { - sum = photon::sat_sub(sum, sum * (now - time) / m_interval); - cnt = photon::sat_sub(cnt, cnt * (now - time) / m_interval); - time = now; - } - } - void put(int64_t val, int64_t add_cnt = 1) { - normalize(); - sum += val; - cnt += add_cnt; - } - void reset() { - sum = 0; - cnt = 0; - time = photon::now; - } - int64_t interval() { return m_interval; } - int64_t interval(int64_t x) { return m_interval = x; } - int64_t val() { - normalize(); - return cnt ? sum / cnt : 0; - } -}; - -class QPSCounter { -public: - int64_t counter = 0; - uint64_t time = photon::now; - uint64_t m_interval = 1UL * 1000 * 1000; - static constexpr uint64_t SEC = 1UL * 1000 * 1000; - - void normalize() { - auto now = photon::now; - if (now - time >= m_interval * 2) { - reset(); - } else if (now - time > m_interval) { - counter = - photon::sat_sub(counter, counter * (now - time) / m_interval); - time = now; - } - } - void put(int64_t val = 1) { - normalize(); - counter += val; - } - void reset() { - counter = 0; - time = photon::now; - } - uint64_t interval() { return m_interval; } - uint64_t interval(uint64_t x) { return m_interval = x; } - int64_t val() { - normalize(); - return counter; - } -}; - -class MaxCounter { -public: - int64_t maxv = 0; - - void put(int64_t val) { - if (val > maxv) { - maxv = val; - } - } - void reset() { maxv = 0; } - int64_t val() { return maxv; } -}; - -class IntervalMaxCounter { -public: - int64_t maxv = 0, last_max = 0; - uint64_t time = 0; - uint64_t m_interval = 5UL * 1000 * 1000; - - void normalize() { - if (photon::now - time >= 2 * m_interval) { - // no `val` or `put` call in 2 intervals - // last interval max must become 0 - reset(); - } else if (photon::now - time > m_interval) { - // one interval passed - // current maxv become certainly max val in last interval - last_max = maxv; - maxv = 0; - time = photon::now; - } - } - - void put(int64_t val) { - normalize(); - maxv = val > maxv ? val : maxv; - } - - void reset() { - maxv = 0; - last_max = 0; - time = photon::now; - } - - uint64_t interval() { return m_interval; } - - uint64_t interval(uint64_t x) { return m_interval = x; } - - int64_t val() { - normalize(); - return maxv > last_max ? maxv : last_max; - } -}; - -template -class LatencyMetric { -public: - LatencyCounter& counter; - uint64_t start; - - explicit LatencyMetric(LatencyCounter& counter) - : counter(counter), start(photon::now) {} - - // no copy or move; - LatencyMetric(LatencyMetric&&) = delete; - LatencyMetric(const LatencyMetric&) = delete; - LatencyMetric& operator=(LatencyMetric&&) = delete; - LatencyMetric& operator=(const LatencyMetric&) = delete; - - ~LatencyMetric() { counter.put(photon::now - start); } -}; - -class AverageLatencyCounter : public AverageCounter { -public: - using MetricType = LatencyMetric; -}; - -class MaxLatencyCounter : public IntervalMaxCounter { -public: - using MetricType = LatencyMetric; -}; - -#define SCOPE_LATENCY(x) \ - std::decay::type::MetricType _CONCAT(__audit_start_time__, \ - __LINE__)(x); - -static ALogLogger default_metrics_logger; - -#define LOG_METRICS(...) (__LOG_METRICS__(ALOG_AUDIT, __VA_ARGS__)) - -#define __LOG_METRICS__(level, first, ...) \ - ({ \ - DEFINE_PROLOGUE(level, prolog); \ - auto __build_lambda__ = [&](ILogOutput* __output_##__LINE__) { \ - if (_IS_LITERAL_STRING(first)) { \ - __log__(level, __output_##__LINE__, prolog, \ - TSTRING(#first).template strip<'\"'>(), \ - ##__VA_ARGS__); \ - } else { \ - __log__(level, __output_##__LINE__, prolog, \ - ConstString::TString<>(), first, ##__VA_ARGS__); \ - } \ - }; \ - LogBuilder( \ - level, std::move(__build_lambda__), \ - &Metric::default_metrics_logger); \ - }) - -#define LOOP_APPEND_METRIC(ret, name) \ - if (!va_##name.empty()) { \ - ret.append(name.help_str()).append("\n"); \ - ret.append(name.type_str()).append("\n"); \ - for (auto x : va_##name) { \ - ret.append( \ - name.render(x.second->val(), nodename.c_str(), x.first)) \ - .append("\n"); \ - } \ - ret.append("\n"); \ - } - -} // namespace Metric diff --git a/src/metrics_fs.h b/src/metrics_fs.h index 194beae0..51f0fc0b 100644 --- a/src/metrics_fs.h +++ b/src/metrics_fs.h @@ -1,55 +1,72 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + #pragma once -#include "metrics.h" -#include +#include #include -#include +struct MetricMeta { + Metric::MaxLatencyCounter latency; + Metric::QPSCounter throughput; + Metric::QPSCounter qps; + Metric::AddCounter total; + Metric::AddCounter interval; -class MetricMeta { -public: - // TODO: add more metrics (error count, virtual block device IO latency, etc...) - Metric::AverageLatencyCounter latency; + MetricMeta() {} }; class MetricFile : public photon::fs::ForwardFile_Ownership { public: MetricMeta *metrics; - MetricFile(IFile *file, MetricMeta *metricMeta) + MetricFile(photon::fs::IFile *file, MetricMeta *metricMeta) : photon::fs::ForwardFile_Ownership(file, true), metrics(metricMeta) {} - virtual ssize_t pread(void *buf, size_t cnt, - off_t offset) override { - auto start = photon::now; - auto ret = m_file->pread(buf, cnt, offset); - if (ret) { - auto duration = photon::now - start; - // latency of read 1MB - metrics->latency.put(duration<<20, ret); + __attribute__((always_inline)) void mark_metrics(ssize_t ret) { + if (ret > 0) { + metrics->throughput.put(ret); + metrics->total.add(ret); + metrics->interval.add(ret); } + } + + virtual ssize_t pread(void *buf, size_t cnt, off_t offset) override { + metrics->qps.put(); + SCOPE_LATENCY(metrics->latency); + auto ret = m_file->pread(buf, cnt, offset); + mark_metrics(ret); return ret; } virtual ssize_t preadv(const struct iovec *iovec, int iovcnt, off_t offset) override { - auto start = photon::now; + metrics->qps.put(); + SCOPE_LATENCY(metrics->latency); auto ret = m_file->preadv(iovec, iovcnt, offset); - if (ret) { - auto duration = photon::now - start; - metrics->latency.put(duration<<20, ret); - } + mark_metrics(ret); return ret; } virtual ssize_t preadv2(const struct iovec *iovec, int iovcnt, off_t offset, int flags) override { - auto start = photon::now; + metrics->qps.put(); + SCOPE_LATENCY(metrics->latency); auto ret = m_file->preadv2(iovec, iovcnt, offset, flags); - if (ret) { - auto duration = photon::now - start; - metrics->latency.put(duration<<20, ret); - } + mark_metrics(ret); return ret; } }; @@ -57,7 +74,8 @@ class MetricFile : public photon::fs::ForwardFile_Ownership { class MetricFS : public photon::fs::ForwardFS_Ownership { public: MetricMeta *metrics; - MetricFS(IFileSystem *fs, MetricMeta *metricMeta) + + MetricFS(photon::fs::IFileSystem *fs, MetricMeta *metricMeta) : photon::fs::ForwardFS_Ownership(fs, true), metrics(metricMeta) {} virtual photon::fs::IFile *open(const char *fn, int flags) override { @@ -72,5 +90,4 @@ class MetricFS : public photon::fs::ForwardFS_Ownership { if (!file) return nullptr; return new MetricFile(file, metrics); } - }; diff --git a/src/textexporter.h b/src/textexporter.h index 0299ca76..0becf225 100644 --- a/src/textexporter.h +++ b/src/textexporter.h @@ -1,3 +1,19 @@ +/* + Copyright The Overlaybd Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + #pragma once #include #include