Merge pull request #1379 from erikoqvist/add_opentelemetry_tracing_sa…

…mpler_configuration

Add opentelemetry tracing sampler configuration

nipap-cli/nipap

@@ -90,19 +90,35 @@ if __name__ == '__main__':
 
     if cfg.has_section("tracing"):
         try:
-            use_grpc = True
+            sampler = None
+
+            if cfg.has_option("tracing", "otel_traces_sampler"):
+                trace_sampler = cfg.get("tracing", "otel_traces_sampler")
+                from opentelemetry.sdk.trace.sampling import _KNOWN_SAMPLERS
+                import dippen
+                if trace_sampler not in _KNOWN_SAMPLERS:
+                    print(
+                        f"Unknown otel_traces_sampler '{trace_sampler}'. Valid samplers are: '{list(_KNOWN_SAMPLERS.keys())}'",
+                        file=sys.stderr)
+                    sys.exit(1)
+                sampler = _KNOWN_SAMPLERS[trace_sampler]
+
             try:
-                otlp_endpoint = cfg.get("tracing", "otlp_grpc_endpoint")
-            except configparser.NoOptionError:
-                # Send trace via nipapd
-                use_grpc = False
-                use_ssl = cfg.getboolean("global", "use_ssl")
-                otlp_endpoint = "https://" if use_ssl else "http://" + (cfg.get("global", "hostname") +
-                                                                        ":" + cfg.get("global", "port") +
-                                                                        "/v1/traces/")
-            tracing.init_tracing("nipap-cli", otlp_endpoint, use_grpc)
-        except KeyError:
-            pass
+                use_grpc = True
+                try:
+                    otlp_endpoint = cfg.get("tracing", "otlp_grpc_endpoint")
+                except configparser.NoOptionError:
+                    # Send trace via nipapd
+                    use_grpc = False
+                    use_ssl = cfg.getboolean("global", "use_ssl")
+                    otlp_endpoint = "https://" if use_ssl else "http://" + (cfg.get("global", "hostname") +
+                                                                            ":" + cfg.get("global", "port") +
+                                                                            "/v1/traces/")
+                tracing.init_tracing("nipap-cli", otlp_endpoint, sampler, use_grpc)
+            except KeyError:
+                pass
+        except ImportError:
+            print("Opentelemetry dependencies are missing. Tracing is not enabled", file=sys.stderr)
 
     # setup our configuration
     nipap_cli.nipap_cli.setup_connection()

nipap-cli/nipaprc

@@ -50,3 +50,5 @@ default_list_vrf_rt = all
 # [tracing]
 # Specify OTLP GRPC endpoint. If no endpoint is specified traces will be sent via nipapd to OpenTelemetry Collector
 # otlp_grpc_endpoint = http://127.0.0.1:4317
+# Set sampler. Valid values are always_on, always_off, parentbased_always_on, parentbased_always_off, traceidratio and parentbased_traceidratio. Default is parentbased_always_on.
+# otel_traces_sampler = always_on

nipap-www/nipapwww/__init__.py

@@ -43,11 +43,20 @@ def create_app(test_config=None):
         try:
             import nipap.tracing
             from opentelemetry.instrumentation.wsgi import OpenTelemetryMiddleware
+            from opentelemetry.sdk.trace.sampling import _KNOWN_SAMPLERS
+
+            sampler = None
+
+            if nipap_config.has_option("tracing", "otel_traces_sampler"):
+                trace_sampler = nipap_config.get("tracing", "otel_traces_sampler")
+                if trace_sampler not in _KNOWN_SAMPLERS:
+                    raise NipapConfigError(f"Unknown otel_traces_sampler '{trace_sampler}'. Valid samplers are: {_KNOWN_SAMPLERS}")
+                sampler = _KNOWN_SAMPLERS[trace_sampler]
 
             if nipap_config.has_option("tracing", "otlp_grpc_endpoint"):
-                nipap.tracing.init_tracing("nipap-www", nipap_config.get("tracing", "otlp_grpc_endpoint"))
+                nipap.tracing.init_tracing("nipap-www", nipap_config.get("tracing", "otlp_grpc_endpoint"), sampler)
             elif nipap_config.has_option("tracing", "otlp_http_endpoint"):
-                nipap.tracing.init_tracing("nipap-www", nipap_config.get("tracing", "otlp_http_endpoint"), False)
+                nipap.tracing.init_tracing("nipap-www", nipap_config.get("tracing", "otlp_http_endpoint"), sampler, False)
             else:
                 raise NipapConfigError("Tracing enabled but no OTLP endpoint configured")
 

nipap/nipap.conf.dist

@@ -208,3 +208,5 @@ secret_key = {{WWW_SECRET_KEY}}
 # otlp_grpc_endpoint=http://opentelemetry-collector:4317
 # Specify OTLP HTTP endpoint. Used to send traces to OpenTelemetry Collector but also used when proxying traces to OpenTelemetry-Collector from nipap-cli
 # otlp_http_endpoint=http://opentelemetry-collector:4318/v1/traces
+# Set sampler. Valid values are always_on, always_off, parentbased_always_on, parentbased_always_off, traceidratio and parentbased_traceidratio. Default is parentbased_always_on.
+# otel_traces_sampler = always_on

nipap/nipap/tracing.py

@@ -10,17 +10,21 @@
     from opentelemetry.sdk.resources import SERVICE_NAME, Resource
     from opentelemetry.sdk.trace import TracerProvider
     from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.sdk.trace.sampling import DEFAULT_ON
     import opentelemetry.exporter.otlp.proto.http.trace_exporter
     from requests import post
 
     tracer = trace.get_tracer("nipap")
 
-    def init_tracing(service_name, endpoint, use_grpc=True):
+    def init_tracing(service_name, endpoint, sampler, use_grpc=True):
         resource = Resource(attributes={
             SERVICE_NAME: service_name
         })
 
-        provider = TracerProvider(resource=resource)
+        if sampler is None:
+            sampler = DEFAULT_ON
+
+        provider = TracerProvider(sampler=sampler, resource=resource)
         if use_grpc:
             processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))
         else:

nipap/nipapd

@@ -233,11 +233,20 @@ if __name__ == '__main__':
             from nipap.tracing import init_tracing, setup
             from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
             from opentelemetry.instrumentation.flask import FlaskInstrumentor
+            from opentelemetry.sdk.trace.sampling import _KNOWN_SAMPLERS
+
+            sampler = None
+
+            if cfg.has_option("tracing", "otel_traces_sampler"):
+                trace_sampler = cfg.get("tracing", "otel_traces_sampler")
+                if trace_sampler not in _KNOWN_SAMPLERS:
+                    raise NipapConfigError(f"Unknown otel_traces_sampler '{trace_sampler}'. Valid samplers are: '{list(_KNOWN_SAMPLERS.keys())}'")
+                sampler = _KNOWN_SAMPLERS[trace_sampler]
 
             if cfg.has_option("tracing", "otlp_grpc_endpoint"):
-                init_tracing("nipapd", cfg.get("tracing", "otlp_grpc_endpoint"))
+                init_tracing("nipapd", cfg.get("tracing", "otlp_grpc_endpoint", sampler))
             elif cfg.has_option("tracing", "otlp_http_endpoint"):
-                init_tracing("nipapd", cfg.get("tracing", "otlp_http_endpoint"), False)
+                init_tracing("nipapd", cfg.get("tracing", "otlp_http_endpoint"), sampler, False)
             else:
                 raise NipapConfigError("Tracing enabled but no OTLP endpoint configured")
 

pynipap/tracing.py

@@ -16,21 +16,26 @@
     from opentelemetry.sdk.resources import SERVICE_NAME, Resource
     from opentelemetry.sdk.trace import TracerProvider
     from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.sdk.trace.sampling import DEFAULT_ON
     import opentelemetry.exporter.otlp.proto.http.trace_exporter
 
     tracer = trace.get_tracer("pynipap")
 
 
-    def init_tracing(service_name, endpoint, use_grpc=True):
+    def init_tracing(service_name, endpoint, sampler, use_grpc=True):
         resource = Resource(attributes={
             SERVICE_NAME: service_name
         })
 
-        provider = TracerProvider(resource=resource)
+        if sampler is None:
+            sampler = DEFAULT_ON
+
+        provider = TracerProvider(sampler=sampler, resource=resource)
         if use_grpc:
             processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint))
         else:
-            processor = BatchSpanProcessor(opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter(endpoint=endpoint))
+            processor = BatchSpanProcessor(
+                opentelemetry.exporter.otlp.proto.http.trace_exporter.OTLPSpanExporter(endpoint=endpoint))
         provider.add_span_processor(processor)
         trace.set_tracer_provider(provider)
 
@@ -70,6 +75,7 @@ def decorated(*args, **kwargs):
                 except BaseException:
                     pass
                 return f(*args, **kwargs)
+
         return decorated
 
 
@@ -99,8 +105,9 @@ def send_content(self, connection, request_body):
                 current_span.set_attribute("net.peer.ip", connection.host)
                 current_span.set_attribute("net.peer.port", connection.port)
                 current_span.set_attribute("net.peer.transport", "ip_tcp")
-                connection.putheader("traceparent", "00-" + hex(current_span.get_span_context().trace_id)[2:].zfill(32) + "-" + hex(
-                    current_span.get_span_context().span_id)[2:].zfill(16) + "-01")
+                connection.putheader("traceparent",
+                                     "00-" + hex(current_span.get_span_context().trace_id)[2:].zfill(32) + "-" + hex(
+                                         current_span.get_span_context().span_id)[2:].zfill(16) + "-0" + ("1" if current_span.is_recording() else "0"))
 
             super().send_content(connection, request_body)
 
@@ -131,8 +138,9 @@ def send_content(self, connection, request_body):
                 current_span.set_attribute("net.peer.ip", connection.host)
                 current_span.set_attribute("net.peer.port", connection.port)
                 current_span.set_attribute("net.peer.transport", "ip_tcp")
-                connection.putheader("traceparent", "00-" + hex(current_span.get_span_context().trace_id)[2:].zfill(32) + "-" + hex(
-                    current_span.get_span_context().span_id)[2:].zfill(16) + "-01")
+                connection.putheader("traceparent",
+                                     "00-" + hex(current_span.get_span_context().trace_id)[2:].zfill(32) + "-" + hex(
+                                         current_span.get_span_context().span_id)[2:].zfill(16) + "-0" + ("1" if current_span.is_recording() else "0"))
 
             super().send_content(connection, request_body)
 
@@ -141,4 +149,8 @@ def create_span(f):
         @wraps(f)
         def decorated(*args, **kwargs):
             return f(*args, **kwargs)
+
         return decorated
+
+    def init_tracing(service_name, endpoint, sampler, use_grpc=True):
+        raise ImportError("Opentelemetry dependencies are missing")