From 347ab06ed4c7e972abaab4751bfb21232bff596d Mon Sep 17 00:00:00 2001 From: Fabrizio Sestito Date: Mon, 9 Dec 2024 19:23:42 +0100 Subject: [PATCH 1/7] build(deps): use opentelmetry-otlp tls feature; add tonic dependency Signed-off-by: Fabrizio Sestito --- Cargo.lock | 3 +++ Cargo.toml | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index b842a339..9e7323c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4279,6 +4279,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-stream", + "tonic", "tower 0.5.1", "tower-http", "tracing", @@ -6315,8 +6316,10 @@ dependencies = [ "percent-encoding", "pin-project", "prost 0.13.4", + "rustls-pemfile 2.2.0", "socket2", "tokio", + "tokio-rustls 0.26.1", "tokio-stream", "tower 0.4.13", "tower-layer", diff --git a/Cargo.toml b/Cargo.toml index a3a34735..19a1e3f1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,11 @@ k8s-openapi = { version = "0.23.0", default-features = false, features = [ lazy_static = "1.4.0" mime = "0.3" num_cpus = "1.16.0" -opentelemetry-otlp = { version = "0.27.0", features = ["metrics", "tonic"] } +opentelemetry-otlp = { version = "0.27.0", features = [ + "metrics", + "tonic", + "tls", +] } opentelemetry = { version = "0.27.0", default-features = false, features = [ "metrics", "trace", @@ -61,6 +65,7 @@ tikv-jemallocator = { version = "0.5.4", features = [ jemalloc_pprof = "0.4.1" tikv-jemalloc-ctl = "0.5.4" rhai = { version = "1.19.0", features = ["sync"] } +tonic = { version = "0.12.3" } [target.'cfg(target_os = "linux")'.dependencies] inotify = "0.11" From 4d198ed0cfe71ca23cef4b74a34af93b1cace721 Mon Sep 17 00:00:00 2001 From: Fabrizio Sestito Date: Mon, 9 Dec 2024 18:52:10 +0100 Subject: [PATCH 2/7] feat: add otlp tls flags Signed-off-by: Fabrizio Sestito --- src/cli.rs | 346 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 193 insertions(+), 153 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 331dbef2..89c61596 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -22,160 +22,200 @@ lazy_static! { pub(crate) fn build_cli() -> Command { let mut args = vec![ - Arg::new("log-level") - .long("log-level") - .value_name("LOG_LEVEL") - .env("KUBEWARDEN_LOG_LEVEL") - .default_value("info") - .value_parser([ - PossibleValue::new("trace"), - PossibleValue::new("debug"), - PossibleValue::new("info"), - PossibleValue::new("warn"), - PossibleValue::new("error"), - ]) - .help("Log level"), - Arg::new("log-fmt") - .long("log-fmt") - .value_name("LOG_FMT") - .env("KUBEWARDEN_LOG_FMT") - .default_value("text") - .value_parser([ - PossibleValue::new("text"), - PossibleValue::new("json"), - PossibleValue::new("otlp"), - ]) - .help("Log output format"), - Arg::new("log-no-color") - .long("log-no-color") - .env("NO_COLOR") - .action(ArgAction::SetTrue) - .help("Disable colored output for logs"), - Arg::new("address") - .long("addr") - .value_name("BIND_ADDRESS") - .default_value("0.0.0.0") - .env("KUBEWARDEN_BIND_ADDRESS") - .help("Bind against ADDRESS"), - Arg::new("port") - .long("port") - .value_name("PORT") - .default_value("3000") - .env("KUBEWARDEN_PORT") - .help("Listen on PORT"), - Arg::new("workers") - .long("workers") - .value_name("WORKERS_NUMBER") - .env("KUBEWARDEN_WORKERS") - .help("Number of workers thread to create"), - Arg::new("cert-file") - .long("cert-file") - .value_name("CERT_FILE") - .default_value("") - .env("KUBEWARDEN_CERT_FILE") - .help("Path to an X.509 certificate file for HTTPS"), - Arg::new("key-file") - .long("key-file") - .value_name("KEY_FILE") - .default_value("") - .env("KUBEWARDEN_KEY_FILE") - .help("Path to an X.509 private key file for HTTPS"), - Arg::new("policies") - .long("policies") - .value_name("POLICIES_FILE") - .env("KUBEWARDEN_POLICIES") - .default_value("policies.yml") - .help("YAML file holding the policies to be loaded and their settings"), - Arg::new("policies-download-dir") - .long("policies-download-dir") - .value_name("POLICIES_DOWNLOAD_DIR") - .default_value(".") - .env("KUBEWARDEN_POLICIES_DOWNLOAD_DIR") - .help("Download path for the policies"), - Arg::new("sigstore-cache-dir") - .long("sigstore-cache-dir") - .value_name("SIGSTORE_CACHE_DIR") - .default_value("sigstore-data") - .env("KUBEWARDEN_SIGSTORE_CACHE_DIR") - .help("Directory used to cache sigstore data"), - Arg::new("sources-path") - .long("sources-path") - .value_name("SOURCES_PATH") - .env("KUBEWARDEN_SOURCES_PATH") - .help("YAML file holding source information (https, registry insecure hosts, custom CA's...)"), - Arg::new("verification-path") - .long("verification-path") - .value_name("VERIFICATION_CONFIG_PATH") - .env("KUBEWARDEN_VERIFICATION_CONFIG_PATH") - .help("YAML file holding verification information (URIs, keys, annotations...)"), - Arg::new("docker-config-json-path") - .long("docker-config-json-path") - .value_name("DOCKER_CONFIG") - .env("KUBEWARDEN_DOCKER_CONFIG_JSON_PATH") - .help("Path to a Docker config.json-like path. Can be used to indicate registry authentication details"), - Arg::new("enable-metrics") - .long("enable-metrics") - .env("KUBEWARDEN_ENABLE_METRICS") - .action(ArgAction::SetTrue) - .help("Enable metrics"), - Arg::new("always-accept-admission-reviews-on-namespace") - .long("always-accept-admission-reviews-on-namespace") - .value_name("NAMESPACE") - .env("KUBEWARDEN_ALWAYS_ACCEPT_ADMISSION_REVIEWS_ON_NAMESPACE") - .required(false) - .help("Always accept AdmissionReviews that target the given namespace"), - Arg::new("disable-timeout-protection") - .long("disable-timeout-protection") - .action(ArgAction::SetTrue) - .env("KUBEWARDEN_DISABLE_TIMEOUT_PROTECTION") - .help("Disable policy timeout protection"), - Arg::new("policy-timeout") - .long("policy-timeout") - .env("KUBEWARDEN_POLICY_TIMEOUT") - .value_name("MAXIMUM_EXECUTION_TIME_SECONDS") - .default_value("2") - .help("Interrupt policy evaluation after the given time"), - Arg::new("daemon") - .long("daemon") - .env("KUBEWARDEN_DAEMON") - .action(ArgAction::SetTrue) - .help("If set, runs policy-server in detached mode as a daemon"), - Arg::new("daemon-pid-file") - .long("daemon-pid-file") - .env("KUBEWARDEN_DAEMON_PID_FILE") - .default_value("policy-server.pid") - .help("Path to pid file, used only when running in daemon mode"), - Arg::new("daemon-stdout-file") - .long("daemon-stdout-file") - .env("KUBEWARDEN_DAEMON_STDOUT_FILE") - .required(false) - .help("Path to file holding stdout, used only when running in daemon mode"), - Arg::new("daemon-stderr-file") - .long("daemon-stderr-file") - .env("KUBEWARDEN_DAEMON_STDERR_FILE") - .required(false) - .help("Path to file holding stderr, used only when running in daemon mode"), - Arg::new("ignore-kubernetes-connection-failure") - .long("ignore-kubernetes-connection-failure") - .env("KUBEWARDEN_IGNORE_KUBERNETES_CONNECTION_FAILURE") - .action(ArgAction::SetTrue) - .help("Do not exit with an error if the Kubernetes connection fails. This will cause context aware policies to break when there's no connection with Kubernetes."), - Arg::new("enable-pprof") - .long("enable-pprof") - .env("KUBEWARDEN_ENABLE_PPROF") - .action(ArgAction::SetTrue) - .help("Enable pprof profiling"), - Arg::new("continue-on-errors") - .long("continue-on-errors") - .env("KUBEWARDEN_CONTINUE_ON_ERRORS") - .action(ArgAction::SetTrue) - .hide(true), - Arg::new("otlp-endpoint") - .long("otlp-endpoint") - .env("OTEL_EXPORTER_OTLP_ENDPOINT") - .default_value("http://localhost:4317") - .help("The OTLP gRPC endpoint for exporting traces and metrics.") + Arg::new("log-level") + .long("log-level") + .value_name("LOG_LEVEL") + .env("KUBEWARDEN_LOG_LEVEL") + .default_value("info") + .value_parser([ + PossibleValue::new("trace"), + PossibleValue::new("debug"), + PossibleValue::new("info"), + PossibleValue::new("warn"), + PossibleValue::new("error"), + ]) + .help("Log level"), + + Arg::new("log-fmt") + .long("log-fmt") + .value_name("LOG_FMT") + .env("KUBEWARDEN_LOG_FMT") + .default_value("text") + .value_parser([ + PossibleValue::new("text"), + PossibleValue::new("json"), + PossibleValue::new("otlp"), + ]) + .help("Log output format"), + + Arg::new("log-no-color") + .long("log-no-color") + .env("NO_COLOR") + .action(ArgAction::SetTrue) + .help("Disable colored output for logs"), + + Arg::new("address") + .long("addr") + .value_name("BIND_ADDRESS") + .default_value("0.0.0.0") + .env("KUBEWARDEN_BIND_ADDRESS") + .help("Bind against ADDRESS"), + + Arg::new("port") + .long("port") + .value_name("PORT") + .default_value("3000") + .env("KUBEWARDEN_PORT") + .help("Listen on PORT"), + + Arg::new("workers") + .long("workers") + .value_name("WORKERS_NUMBER") + .env("KUBEWARDEN_WORKERS") + .help("Number of worker threads to create"), + + Arg::new("cert-file") + .long("cert-file") + .value_name("CERT_FILE") + .default_value("") + .env("KUBEWARDEN_CERT_FILE") + .help("Path to an X.509 certificate file for HTTPS"), + + Arg::new("key-file") + .long("key-file") + .value_name("KEY_FILE") + .default_value("") + .env("KUBEWARDEN_KEY_FILE") + .help("Path to an X.509 private key file for HTTPS"), + + Arg::new("policies") + .long("policies") + .value_name("POLICIES_FILE") + .env("KUBEWARDEN_POLICIES") + .default_value("policies.yml") + .help("YAML file holding the policies to be loaded and their settings"), + + Arg::new("policies-download-dir") + .long("policies-download-dir") + .value_name("POLICIES_DOWNLOAD_DIR") + .default_value(".") + .env("KUBEWARDEN_POLICIES_DOWNLOAD_DIR") + .help("Download path for the policies"), + + Arg::new("sigstore-cache-dir") + .long("sigstore-cache-dir") + .value_name("SIGSTORE_CACHE_DIR") + .default_value("sigstore-data") + .env("KUBEWARDEN_SIGSTORE_CACHE_DIR") + .help("Directory used to cache sigstore data"), + + Arg::new("sources-path") + .long("sources-path") + .value_name("SOURCES_PATH") + .env("KUBEWARDEN_SOURCES_PATH") + .help("YAML file holding source information (https, registry insecure hosts, custom CA's...)"), + + Arg::new("verification-path") + .long("verification-path") + .value_name("VERIFICATION_CONFIG_PATH") + .env("KUBEWARDEN_VERIFICATION_CONFIG_PATH") + .help("YAML file holding verification information (URIs, keys, annotations...)"), + + Arg::new("docker-config-json-path") + .long("docker-config-json-path") + .value_name("DOCKER_CONFIG") + .env("KUBEWARDEN_DOCKER_CONFIG_JSON_PATH") + .help("Path to a Docker config.json-like path. Can be used to indicate registry authentication details"), + + Arg::new("enable-metrics") + .long("enable-metrics") + .env("KUBEWARDEN_ENABLE_METRICS") + .action(ArgAction::SetTrue) + .help("Enable metrics"), + + Arg::new("always-accept-admission-reviews-on-namespace") + .long("always-accept-admission-reviews-on-namespace") + .value_name("NAMESPACE") + .env("KUBEWARDEN_ALWAYS_ACCEPT_ADMISSION_REVIEWS_ON_NAMESPACE") + .required(false) + .help("Always accept AdmissionReviews that target the given namespace"), + + Arg::new("disable-timeout-protection") + .long("disable-timeout-protection") + .action(ArgAction::SetTrue) + .env("KUBEWARDEN_DISABLE_TIMEOUT_PROTECTION") + .help("Disable policy timeout protection"), + + Arg::new("policy-timeout") + .long("policy-timeout") + .env("KUBEWARDEN_POLICY_TIMEOUT") + .value_name("MAXIMUM_EXECUTION_TIME_SECONDS") + .default_value("2") + .help("Interrupt policy evaluation after the given time"), + + Arg::new("daemon") + .long("daemon") + .env("KUBEWARDEN_DAEMON") + .action(ArgAction::SetTrue) + .help("If set, runs policy-server in detached mode as a daemon"), + + Arg::new("daemon-pid-file") + .long("daemon-pid-file") + .env("KUBEWARDEN_DAEMON_PID_FILE") + .default_value("policy-server.pid") + .help("Path to the PID file, used only when running in daemon mode"), + + Arg::new("daemon-stdout-file") + .long("daemon-stdout-file") + .env("KUBEWARDEN_DAEMON_STDOUT_FILE") + .required(false) + .help("Path to the file holding stdout, used only when running in daemon mode"), + + Arg::new("daemon-stderr-file") + .long("daemon-stderr-file") + .env("KUBEWARDEN_DAEMON_STDERR_FILE") + .required(false) + .help("Path to the file holding stderr, used only when running in daemon mode"), + + Arg::new("ignore-kubernetes-connection-failure") + .long("ignore-kubernetes-connection-failure") + .env("KUBEWARDEN_IGNORE_KUBERNETES_CONNECTION_FAILURE") + .action(ArgAction::SetTrue) + .help("Do not exit with an error if the Kubernetes connection fails. This will cause context-aware policies to break when there's no connection with Kubernetes."), + + Arg::new("enable-pprof") + .long("enable-pprof") + .env("KUBEWARDEN_ENABLE_PPROF") + .action(ArgAction::SetTrue) + .help("Enable pprof profiling"), + + Arg::new("continue-on-errors") + .long("continue-on-errors") + .env("KUBEWARDEN_CONTINUE_ON_ERRORS") + .action(ArgAction::SetTrue) + .hide(true), + + Arg::new("otlp-endpoint") + .long("otlp-endpoint") + .env("OTEL_EXPORTER_OTLP_ENDPOINT") + .help("The OTLP gRPC endpoint for exporting traces and metrics."), + + Arg::new("otlp-certificate") + .long("otlp-certificate") + .env("OTEL_EXPORTER_OTLP_CERTIFICATE") + .help("Path to the trusted certificate in PEM format used for verifying the TLS credentials of the OTLP gRPC endpoint."), + + Arg::new("otlp-client-certificate") + .long("otlp-client-certificate") + .env("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE") + .help("Path to the certificate in PEM format to use in mTLS communication."), + + Arg::new("otlp-client-key") + .long("otlp-client-key") + .env("OTEL_EXPORTER_OTLP_CLIENT_KEY") + .help("Path to the client private key in PEM format to use in mTLS communication."), ]; + args.sort_by(|a, b| a.get_id().cmp(b.get_id())); Command::new(crate_name!()) From 6192f11c93491fc7f6bc3dd026a0398f1f772f4c Mon Sep 17 00:00:00 2001 From: Fabrizio Sestito Date: Mon, 9 Dec 2024 18:52:26 +0100 Subject: [PATCH 3/7] feat: add otlp tls config Signed-off-by: Fabrizio Sestito --- src/config.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/config.rs b/src/config.rs index 49e5ffc0..97a6aee7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -42,6 +42,7 @@ pub struct Config { pub log_fmt: String, pub log_no_color: bool, pub otlp_endpoint: Option, + pub otlp_tls_config: OtlpTlsConfig, pub daemon: bool, pub enable_pprof: bool, pub daemon_pid_file: String, @@ -55,6 +56,44 @@ pub struct TlsConfig { pub key_file: String, } +#[derive(Clone, Default)] +pub struct OtlpTlsConfig { + pub ca_file: Option, + pub cert_file: Option, + pub key_file: Option, +} + +impl TryFrom for tonic::transport::ClientTlsConfig { + type Error = anyhow::Error; + + fn try_from(value: OtlpTlsConfig) -> Result { + use std::fs; + use tonic::transport::{Certificate, ClientTlsConfig, Identity}; + + let mut tls = ClientTlsConfig::new(); + + if let Some(ca) = value.ca_file { + let ca_cert = fs::read(ca)?; + tls = tls.ca_certificate(Certificate::from_pem(ca_cert)) + } + + if let Some(cert) = value.cert_file { + let cert = fs::read(cert)?; + + let key = value + .key_file + .map(fs::read) + .transpose()? + .unwrap_or_default(); + + let identity = Identity::from_pem(cert, key); + tls = tls.identity(identity); + } + + Ok(tls) + } +} + impl Config { pub fn from_args(matches: &ArgMatches) -> Result { // init some variables based on the cli parameters @@ -128,6 +167,17 @@ impl Config { .to_owned(); let otlp_endpoint = matches.get_one::("otlp-endpoint").cloned(); + let otlp_ca_file = matches.get_one::("otlp-certificate").cloned(); + let otlp_cert_file = matches + .get_one::("otlp-client-certificate") + .cloned(); + let otlp_key_file = matches.get_one::("otlp-client-key").cloned(); + + let otlp_tls_config = OtlpTlsConfig { + ca_file: otlp_ca_file, + cert_file: otlp_cert_file, + key_file: otlp_key_file, + }; let (cert_file, key_file) = tls_files(matches)?; let tls_config = if cert_file.is_empty() { @@ -165,6 +215,7 @@ impl Config { log_fmt, log_no_color, otlp_endpoint, + otlp_tls_config, daemon, daemon_pid_file, daemon_stdout_file, From 65cdfb2195e80f42f958bf79d6d91a385ad8d3a0 Mon Sep 17 00:00:00 2001 From: Fabrizio Sestito Date: Mon, 9 Dec 2024 18:52:52 +0100 Subject: [PATCH 4/7] feat: initialize metrics and tracing with tls config Signed-off-by: Fabrizio Sestito --- src/main.rs | 6 +++++- src/metrics.rs | 7 +++++-- src/tracing.rs | 14 ++++++++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/main.rs b/src/main.rs index 0fe86cee..505d806b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,10 +26,14 @@ async fn main() -> Result<()> { &config.log_fmt, config.log_no_color, config.otlp_endpoint.as_deref(), + config.otlp_tls_config.clone(), )?; if config.metrics_enabled { - setup_metrics(config.otlp_endpoint.as_deref())?; + setup_metrics( + config.otlp_endpoint.as_deref(), + config.otlp_tls_config.clone(), + )?; }; if config.daemon { diff --git a/src/metrics.rs b/src/metrics.rs index 3342d775..38b38669 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -1,6 +1,6 @@ use anyhow::Result; use opentelemetry::{global, KeyValue}; -use opentelemetry_otlp::{ExportConfig, WithExportConfig}; +use opentelemetry_otlp::{ExportConfig, WithExportConfig, WithTonicConfig}; use opentelemetry_sdk::runtime; mod policy_evaluations_total; @@ -8,11 +8,14 @@ pub use policy_evaluations_total::add_policy_evaluation; mod policy_evaluations_latency; pub use policy_evaluations_latency::record_policy_latency; +use crate::config::OtlpTlsConfig; + const METER_NAME: &str = "kubewarden"; -pub fn setup_metrics(otlp_endpoint: Option<&str>) -> Result<()> { +pub fn setup_metrics(otlp_endpoint: Option<&str>, otlp_tls_config: OtlpTlsConfig) -> Result<()> { let mut metric_exporter_builder = opentelemetry_otlp::MetricExporter::builder() .with_tonic() + .with_tls_config(otlp_tls_config.try_into()?) .with_export_config(ExportConfig::default()); if let Some(endpoint) = otlp_endpoint { metric_exporter_builder = metric_exporter_builder.with_endpoint(endpoint); diff --git a/src/tracing.rs b/src/tracing.rs index 8fc4e098..e0068159 100644 --- a/src/tracing.rs +++ b/src/tracing.rs @@ -1,10 +1,13 @@ +use std::convert::TryInto; + use anyhow::{anyhow, Result}; use opentelemetry::trace::TracerProvider; -use opentelemetry_otlp::WithExportConfig; +use opentelemetry_otlp::{WithExportConfig, WithTonicConfig}; + use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; -use crate::config; +use crate::config::{self, OtlpTlsConfig}; // Setup the tracing system. This MUST be done inside of a tokio Runtime // because some collectors rely on it and would panic otherwise. @@ -13,6 +16,7 @@ pub fn setup_tracing( log_fmt: &str, log_no_color: bool, otlp_endpoint: Option<&str>, + otlp_tls_config: OtlpTlsConfig, ) -> Result<()> { // setup logging let filter_layer = EnvFilter::new(log_level) @@ -45,8 +49,10 @@ pub fn setup_tracing( // OpenTelemetry collector using the OTLP format. // If no endpoint is provided, the default one is used. // The default endpoint is "http://localhost:4317". - let mut otlp_exporter_builder = - opentelemetry_otlp::SpanExporter::builder().with_tonic(); + // + let mut otlp_exporter_builder = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_tls_config(otlp_tls_config.try_into()?); if let Some(endpoint) = otlp_endpoint { otlp_exporter_builder = otlp_exporter_builder.with_endpoint(endpoint); From a4699aca7ecd128bcf437b4d02ce0d01681541a7 Mon Sep 17 00:00:00 2001 From: Fabrizio Sestito Date: Tue, 10 Dec 2024 12:36:09 +0100 Subject: [PATCH 5/7] build(deps): add rcgen to dev dependencies Signed-off-by: Fabrizio Sestito --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 19a1e3f1..2b842e54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,9 +79,9 @@ tower = { version = "0.5", features = ["util"] } http-body-util = "0.1.1" testcontainers = { version = "0.23", features = ["watchdog"] } backon = { version = "1.3", features = ["tokio-sleep"] } +rcgen = { version = "0.13", features = ["crypto"] } [target.'cfg(target_os = "linux")'.dev-dependencies] -rcgen = { version = "0.13", features = ["crypto"] } openssl = "0.10" reqwest = { version = "0.12", default-features = false, features = [ "charset", From d7e156e9cbfa4d52490befb044ce785db2abfc23 Mon Sep 17 00:00:00 2001 From: Fabrizio Sestito Date: Mon, 9 Dec 2024 18:53:05 +0100 Subject: [PATCH 6/7] test: update otel integration tests Signed-off-by: Fabrizio Sestito --- tests/common/mod.rs | 1 + tests/data/otel-collector-config.yaml | 5 ++ tests/integration_test.rs | 107 +++++++++++++++++++++++--- 3 files changed, 102 insertions(+), 11 deletions(-) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index dd9c22a8..bba7c034 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -121,6 +121,7 @@ pub(crate) fn default_test_config() -> Config { log_fmt: "json".to_owned(), log_no_color: false, otlp_endpoint: None, + otlp_tls_config: Default::default(), daemon: false, daemon_pid_file: "policy_server.pid".to_owned(), daemon_stdout_file: None, diff --git a/tests/data/otel-collector-config.yaml b/tests/data/otel-collector-config.yaml index c2bc8947..8531a8f4 100644 --- a/tests/data/otel-collector-config.yaml +++ b/tests/data/otel-collector-config.yaml @@ -5,6 +5,11 @@ receivers: otlp: protocols: grpc: + tls: + ca_file: "certs/server-ca.pem" + cert_file: "certs/server-cert.pem" + key_file: "certs/server-key.pem" + client_ca_file: "certs/client-ca.pem" exporters: file/metrics: diff --git a/tests/integration_test.rs b/tests/integration_test.rs index d9f69150..6d53a67c 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -23,12 +23,14 @@ use policy_evaluator::{ admission_response::AdmissionResponseStatus, policy_fetcher::verify::config::VerificationConfigV1, }; +use policy_server::config::OtlpTlsConfig; use policy_server::{ api::admission_review::AdmissionReviewResponse, config::{PolicyMode, PolicyOrPolicyGroup}, metrics::setup_metrics, tracing::setup_tracing, }; +use rcgen::{BasicConstraints, CertificateParams, DnType, IsCa, KeyPair}; use regex::Regex; use rstest::*; use tempfile::NamedTempFile; @@ -37,6 +39,7 @@ use testcontainers::{ runners::AsyncRunner, GenericImage, ImageExt, }; +use tokio::fs; use tower::ServiceExt; use crate::common::default_test_config; @@ -746,18 +749,51 @@ async fn test_detect_certificate_rotation() { async fn test_otel() { setup(); - let mut otelc_config_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - otelc_config_path.push("tests/data/otel-collector-config.yaml"); + let otelc_config_path = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/otel-collector-config.yaml"); - let metrics_output_file = NamedTempFile::new().unwrap(); - let metrics_output_file_path = metrics_output_file.path(); + let (server_ca, server_cert, server_key) = generate_tls_certs(); + let (client_ca, client_cert, client_key) = generate_tls_certs(); + + let server_ca_file = NamedTempFile::new().unwrap(); + let server_cert_file = NamedTempFile::new().unwrap(); + let server_key_file = NamedTempFile::new().unwrap(); + + let client_ca_file = NamedTempFile::new().unwrap(); + let client_cert_file = NamedTempFile::new().unwrap(); + let client_key_file = NamedTempFile::new().unwrap(); + + let files_and_contents = [ + (server_ca_file.path(), &server_ca), + (server_cert_file.path(), &server_cert), + (server_key_file.path(), &server_key), + (client_ca_file.path(), &client_ca), + (client_cert_file.path(), &client_cert), + (client_key_file.path(), &client_key), + ]; + + for (file_path, content) in &files_and_contents { + fs::write(file_path, content).await.unwrap(); + } + let metrics_output_file = NamedTempFile::new().unwrap(); let traces_output_file = NamedTempFile::new().unwrap(); - let traces_output_file_path = traces_output_file.path(); let permissions = Permissions::from_mode(0o666); - set_permissions(metrics_output_file_path, permissions.clone()).unwrap(); - set_permissions(traces_output_file_path, permissions).unwrap(); + let files_to_set_permissions = [ + metrics_output_file.path(), + traces_output_file.path(), + server_ca_file.path(), + server_cert_file.path(), + server_key_file.path(), + client_ca_file.path(), + client_cert_file.path(), + client_key_file.path(), + ]; + + for file_path in &files_to_set_permissions { + set_permissions(file_path, permissions.clone()).unwrap(); + } let otelc = GenericImage::new("otel/opentelemetry-collector", "0.98.0") .with_wait_for(WaitFor::message_on_stderr("Everything is ready")) @@ -766,13 +802,29 @@ async fn test_otel() { "/etc/otel-collector-config.yaml", )) .with_mount(Mount::bind_mount( - metrics_output_file_path.to_str().unwrap(), + metrics_output_file.path().to_str().unwrap(), "/tmp/metrics.json", )) .with_mount(Mount::bind_mount( - traces_output_file_path.to_str().unwrap(), + traces_output_file.path().to_str().unwrap(), "/tmp/traces.json", )) + .with_mount(Mount::bind_mount( + server_ca_file.path().to_str().unwrap(), + "/certs/server-ca.pem", + )) + .with_mount(Mount::bind_mount( + server_cert_file.path().to_str().unwrap(), + "/certs/server-cert.pem", + )) + .with_mount(Mount::bind_mount( + server_key_file.path().to_str().unwrap(), + "/certs/server-key.pem", + )) + .with_mount(Mount::bind_mount( + client_ca_file.path().to_str().unwrap(), + "/certs/client-ca.pem", + )) .with_mapped_port(1337, 4317.into()) .with_cmd(vec!["--config=/etc/otel-collector-config.yaml"]) .with_startup_timeout(Duration::from_secs(30)) @@ -783,13 +835,24 @@ async fn test_otel() { let mut config = default_test_config(); config.metrics_enabled = true; config.log_fmt = "otlp".to_string(); - config.otlp_endpoint = Some("http://localhost:1337".to_string()); - setup_metrics(config.otlp_endpoint.as_deref()).unwrap(); + config.otlp_endpoint = Some("https://localhost:1337".to_string()); + config.otlp_tls_config = OtlpTlsConfig { + ca_file: Some(server_ca_file.path().to_owned()), + cert_file: Some(client_cert_file.path().to_owned()), + key_file: Some(client_key_file.path().to_owned()), + }; + + setup_metrics( + config.otlp_endpoint.as_deref(), + config.otlp_tls_config.clone(), + ) + .unwrap(); setup_tracing( &config.log_level, &config.log_fmt, config.log_no_color, config.otlp_endpoint.as_deref(), + config.otlp_tls_config.clone(), ) .unwrap(); @@ -862,3 +925,25 @@ async fn parse_exporter_output( serde_json::from_str(&exporter_output) } + +fn generate_tls_certs() -> (String, String, String) { + let ca_key = KeyPair::generate().unwrap(); + let mut params = CertificateParams::new(vec!["My Test CA".to_string()]).unwrap(); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca_cert = params.self_signed(&ca_key).unwrap(); + let ca_cert_pem = ca_cert.pem(); + + let mut params = CertificateParams::new(vec!["localhost".to_string()]).unwrap(); + params + .distinguished_name + .push(DnType::OrganizationName, "Kubewarden"); + params + .distinguished_name + .push(DnType::CommonName, "kubewarden.io"); + + let cert_key = KeyPair::generate().unwrap(); + let cert = params.signed_by(&cert_key, &ca_cert, &ca_key).unwrap(); + let key = cert_key.serialize_pem(); + + (ca_cert_pem, cert.pem(), key) +} From f0879b11a2aadb68cb394eeea401a48fb18d4f29 Mon Sep 17 00:00:00 2001 From: Fabrizio Sestito Date: Wed, 11 Dec 2024 08:31:46 +0100 Subject: [PATCH 7/7] refactor: use OTLP env variables only Signed-off-by: Fabrizio Sestito --- src/cli.rs | 20 -------- src/config.rs | 101 +++++++++++++++++--------------------- src/main.rs | 13 +---- src/metrics.rs | 19 ++++--- src/tracing.rs | 25 +++------- tests/common/mod.rs | 2 - tests/integration_test.rs | 38 +++++++------- 7 files changed, 79 insertions(+), 139 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 89c61596..85efeb51 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -194,26 +194,6 @@ pub(crate) fn build_cli() -> Command { .env("KUBEWARDEN_CONTINUE_ON_ERRORS") .action(ArgAction::SetTrue) .hide(true), - - Arg::new("otlp-endpoint") - .long("otlp-endpoint") - .env("OTEL_EXPORTER_OTLP_ENDPOINT") - .help("The OTLP gRPC endpoint for exporting traces and metrics."), - - Arg::new("otlp-certificate") - .long("otlp-certificate") - .env("OTEL_EXPORTER_OTLP_CERTIFICATE") - .help("Path to the trusted certificate in PEM format used for verifying the TLS credentials of the OTLP gRPC endpoint."), - - Arg::new("otlp-client-certificate") - .long("otlp-client-certificate") - .env("OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE") - .help("Path to the certificate in PEM format to use in mTLS communication."), - - Arg::new("otlp-client-key") - .long("otlp-client-key") - .env("OTEL_EXPORTER_OTLP_CLIENT_KEY") - .help("Path to the client private key in PEM format to use in mTLS communication."), ]; args.sort_by(|a, b| a.get_id().cmp(b.get_id())); diff --git a/src/config.rs b/src/config.rs index 97a6aee7..fd2962f9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,10 +12,11 @@ use serde::Deserialize; use std::{ collections::{BTreeSet, HashMap}, env, - fs::File, + fs::{self, File}, net::SocketAddr, path::{Path, PathBuf}, }; +use tonic::transport::{Certificate, ClientTlsConfig, Identity}; pub static SERVICE_NAME: &str = "kubewarden-policy-server"; const DOCKER_CONFIG_ENV_VAR: &str = "DOCKER_CONFIG"; @@ -41,8 +42,6 @@ pub struct Config { pub log_level: String, pub log_fmt: String, pub log_no_color: bool, - pub otlp_endpoint: Option, - pub otlp_tls_config: OtlpTlsConfig, pub daemon: bool, pub enable_pprof: bool, pub daemon_pid_file: String, @@ -56,44 +55,6 @@ pub struct TlsConfig { pub key_file: String, } -#[derive(Clone, Default)] -pub struct OtlpTlsConfig { - pub ca_file: Option, - pub cert_file: Option, - pub key_file: Option, -} - -impl TryFrom for tonic::transport::ClientTlsConfig { - type Error = anyhow::Error; - - fn try_from(value: OtlpTlsConfig) -> Result { - use std::fs; - use tonic::transport::{Certificate, ClientTlsConfig, Identity}; - - let mut tls = ClientTlsConfig::new(); - - if let Some(ca) = value.ca_file { - let ca_cert = fs::read(ca)?; - tls = tls.ca_certificate(Certificate::from_pem(ca_cert)) - } - - if let Some(cert) = value.cert_file { - let cert = fs::read(cert)?; - - let key = value - .key_file - .map(fs::read) - .transpose()? - .unwrap_or_default(); - - let identity = Identity::from_pem(cert, key); - tls = tls.identity(identity); - } - - Ok(tls) - } -} - impl Config { pub fn from_args(matches: &ArgMatches) -> Result { // init some variables based on the cli parameters @@ -166,19 +127,6 @@ impl Config { .expect("clap should have assigned a default value") .to_owned(); - let otlp_endpoint = matches.get_one::("otlp-endpoint").cloned(); - let otlp_ca_file = matches.get_one::("otlp-certificate").cloned(); - let otlp_cert_file = matches - .get_one::("otlp-client-certificate") - .cloned(); - let otlp_key_file = matches.get_one::("otlp-client-key").cloned(); - - let otlp_tls_config = OtlpTlsConfig { - ca_file: otlp_ca_file, - cert_file: otlp_cert_file, - key_file: otlp_key_file, - }; - let (cert_file, key_file) = tls_files(matches)?; let tls_config = if cert_file.is_empty() { None @@ -214,8 +162,6 @@ impl Config { log_level, log_fmt, log_no_color, - otlp_endpoint, - otlp_tls_config, daemon, daemon_pid_file, daemon_stdout_file, @@ -482,6 +428,49 @@ fn read_policies_file(path: &Path) -> Result Result { + let mut client_tls_config = ClientTlsConfig::new(); + + let ca_env = format!("OTEL_EXPORTER_OTLP_{}CERTIFICATE", prefix); + let fallback_ca_env = "OTEL_EXPORTER_OTLP_CERTIFICATE"; + + let ca_file = env::var(&ca_env) + .or_else(|_| env::var(fallback_ca_env)) + .ok(); + + if let Some(ca_path) = ca_file { + let ca_cert = std::fs::read(ca_path)?; + client_tls_config = client_tls_config.ca_certificate(Certificate::from_pem(ca_cert)); + } + + let client_cert_env = format!("OTEL_EXPORTER_OTLP_{}CLIENT_CERTIFICATE", prefix); + let fallback_client_cert_env = "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE"; + + let client_cert_file = std::env::var(&client_cert_env) + .or_else(|_| std::env::var(fallback_client_cert_env)) + .ok(); + + let client_key_env = format!("OTEL_EXPORTER_OTLP_{}CLIENT_KEY", prefix); + let fallback_client_key_env = "OTEL_EXPORTER_OTLP_CLIENT_KEY"; + + let client_key_file = std::env::var(&client_key_env) + .or_else(|_| std::env::var(fallback_client_key_env)) + .ok(); + + if let (Some(cert_path), Some(key_path)) = (client_cert_file, client_key_file) { + let cert = fs::read(cert_path)?; + let key = fs::read(key_path)?; + + let identity = Identity::from_pem(cert, key); + client_tls_config = client_tls_config.identity(identity); + } + + Ok(client_tls_config) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/main.rs b/src/main.rs index 505d806b..045985c2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,19 +21,10 @@ async fn main() -> Result<()> { let matches = cli::build_cli().get_matches(); let config = policy_server::config::Config::from_args(&matches)?; - setup_tracing( - &config.log_level, - &config.log_fmt, - config.log_no_color, - config.otlp_endpoint.as_deref(), - config.otlp_tls_config.clone(), - )?; + setup_tracing(&config.log_level, &config.log_fmt, config.log_no_color)?; if config.metrics_enabled { - setup_metrics( - config.otlp_endpoint.as_deref(), - config.otlp_tls_config.clone(), - )?; + setup_metrics()?; }; if config.daemon { diff --git a/src/metrics.rs b/src/metrics.rs index 38b38669..d9e1a2bd 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -8,21 +8,20 @@ pub use policy_evaluations_total::add_policy_evaluation; mod policy_evaluations_latency; pub use policy_evaluations_latency::record_policy_latency; -use crate::config::OtlpTlsConfig; +use crate::config::build_client_tls_config_from_env; const METER_NAME: &str = "kubewarden"; -pub fn setup_metrics(otlp_endpoint: Option<&str>, otlp_tls_config: OtlpTlsConfig) -> Result<()> { - let mut metric_exporter_builder = opentelemetry_otlp::MetricExporter::builder() +pub fn setup_metrics() -> Result<()> { + let metric_exporter = opentelemetry_otlp::MetricExporter::builder() .with_tonic() - .with_tls_config(otlp_tls_config.try_into()?) - .with_export_config(ExportConfig::default()); - if let Some(endpoint) = otlp_endpoint { - metric_exporter_builder = metric_exporter_builder.with_endpoint(endpoint); - } - let meter_reader = metric_exporter_builder.build()?; + .with_tls_config(build_client_tls_config_from_env("METRICS")?) + .with_export_config(ExportConfig::default()) + .build()?; + let periodic_reader = - opentelemetry_sdk::metrics::PeriodicReader::builder(meter_reader, runtime::Tokio).build(); + opentelemetry_sdk::metrics::PeriodicReader::builder(metric_exporter, runtime::Tokio) + .build(); let meter_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder() .with_reader(periodic_reader) .build(); diff --git a/src/tracing.rs b/src/tracing.rs index e0068159..44d7338e 100644 --- a/src/tracing.rs +++ b/src/tracing.rs @@ -1,23 +1,15 @@ -use std::convert::TryInto; - use anyhow::{anyhow, Result}; use opentelemetry::trace::TracerProvider; -use opentelemetry_otlp::{WithExportConfig, WithTonicConfig}; +use opentelemetry_otlp::WithTonicConfig; use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; -use crate::config::{self, OtlpTlsConfig}; +use crate::config::{self, build_client_tls_config_from_env}; // Setup the tracing system. This MUST be done inside of a tokio Runtime // because some collectors rely on it and would panic otherwise. -pub fn setup_tracing( - log_level: &str, - log_fmt: &str, - log_no_color: bool, - otlp_endpoint: Option<&str>, - otlp_tls_config: OtlpTlsConfig, -) -> Result<()> { +pub fn setup_tracing(log_level: &str, log_fmt: &str, log_no_color: bool) -> Result<()> { // setup logging let filter_layer = EnvFilter::new(log_level) // some of our dependencies generate trace events too, but we don't care about them -> @@ -50,15 +42,10 @@ pub fn setup_tracing( // If no endpoint is provided, the default one is used. // The default endpoint is "http://localhost:4317". // - let mut otlp_exporter_builder = opentelemetry_otlp::SpanExporter::builder() + let otlp_exporter = opentelemetry_otlp::SpanExporter::builder() .with_tonic() - .with_tls_config(otlp_tls_config.try_into()?); - - if let Some(endpoint) = otlp_endpoint { - otlp_exporter_builder = otlp_exporter_builder.with_endpoint(endpoint); - } - - let otlp_exporter = otlp_exporter_builder.build()?; + .with_tls_config(build_client_tls_config_from_env("OTLP")?) + .build()?; let tracer_provider = opentelemetry_sdk::trace::TracerProvider::builder() .with_resource(opentelemetry_sdk::Resource::new(vec![ diff --git a/tests/common/mod.rs b/tests/common/mod.rs index bba7c034..7b32c9d2 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -120,8 +120,6 @@ pub(crate) fn default_test_config() -> Config { log_level: "info".to_owned(), log_fmt: "json".to_owned(), log_no_color: false, - otlp_endpoint: None, - otlp_tls_config: Default::default(), daemon: false, daemon_pid_file: "policy_server.pid".to_owned(), daemon_stdout_file: None, diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 6d53a67c..07116614 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -23,7 +23,6 @@ use policy_evaluator::{ admission_response::AdmissionResponseStatus, policy_fetcher::verify::config::VerificationConfigV1, }; -use policy_server::config::OtlpTlsConfig; use policy_server::{ api::admission_review::AdmissionReviewResponse, config::{PolicyMode, PolicyOrPolicyGroup}, @@ -832,29 +831,26 @@ async fn test_otel() { .await .unwrap(); + std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", "https://localhost:1337"); + std::env::set_var( + "OTEL_EXPORTER_OTLP_CERTIFICATE", + server_ca_file.path().to_str().unwrap(), + ); + std::env::set_var( + "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", + client_cert_file.path().to_str().unwrap(), + ); + std::env::set_var( + "OTEL_EXPORTER_OTLP_CLIENT_KEY", + client_key_file.path().to_str().unwrap(), + ); + let mut config = default_test_config(); config.metrics_enabled = true; config.log_fmt = "otlp".to_string(); - config.otlp_endpoint = Some("https://localhost:1337".to_string()); - config.otlp_tls_config = OtlpTlsConfig { - ca_file: Some(server_ca_file.path().to_owned()), - cert_file: Some(client_cert_file.path().to_owned()), - key_file: Some(client_key_file.path().to_owned()), - }; - - setup_metrics( - config.otlp_endpoint.as_deref(), - config.otlp_tls_config.clone(), - ) - .unwrap(); - setup_tracing( - &config.log_level, - &config.log_fmt, - config.log_no_color, - config.otlp_endpoint.as_deref(), - config.otlp_tls_config.clone(), - ) - .unwrap(); + + setup_metrics().unwrap(); + setup_tracing(&config.log_level, &config.log_fmt, config.log_no_color).unwrap(); let app = app(config).await;