Profiling (#327)

* feat: initial profiling impl

* feat: better options

* chore: add todo

* add tests

* add debug logs

* fix logging, add additional warnings

* readd nonzero location and function ids, add stack frame count

* fix: format

* remove unused imports

* add stop_profiling, filtering of internal threads

* feat: handle process forking, use logger provider

* chore: get rid of unnecessary imports

* stacktrace building without line lookups

* account for processing time when scheduling next sampler sleep

* remove debug printout

* tear down profiler thread on exit

* hack around process hang on exit

* hack around shutdown

* get rid of logger provider dude to windows hangs

* add check for register_at_fork

* readd logger provider

* add comment about trace context to profiling tests

* rename call_stack_interval to call_stack_interval_millis

* remove unused import

* pass explicit thread_states to to_log_record / encode_cpu_profile

* remove access to _profiler from _start_profiler_thread

* wrap context hooks

* refactor Profiler into a class with methods (#355)

* lint

---------

Co-authored-by: Pablo Collins <pablo.collins@gmail.com>

.pylintrc

@@ -169,7 +169,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=zipkin_pb2.*
+generated-members=zipkin_pb2.*,profile_pb2.*
 
 # Tells whether missing members accessed in mixin class should be ignored. A
 # mixin class is detected if its name ends with "mixin" (case insensitive).
@@ -439,7 +439,8 @@ exclude-protected=_asdict,
                   _replace,
                   _source,
                   _make,
-                  _Span
+                  _Span,
+                  _current_frames
 
 # List of valid names for the first argument in a class method.
 valid-classmethod-first-arg=cls
@@ -463,7 +464,7 @@ max-bool-expr=5
 max-branches=12
 
 # Maximum number of locals for function / method body.
-max-locals=15
+max-locals=32
 
 # Maximum number of parents for a class (see R0901).
 max-parents=7
@@ -475,7 +476,7 @@ max-public-methods=20
 max-returns=6
 
 # Maximum number of statements in function / method body.
-max-statements=50
+max-statements=100
 
 # Minimum number of public methods for a class (see R0903).
 min-public-methods=2

DEVELOPMENT.md

@@ -26,7 +26,7 @@ the test project's environment. Assuming the test project environment lives at
 version of package in the test project.
 
 ```
-make develop DEV_ENV=/path/to/test/project/venv
+make develop DEV_VENV=/path/to/test/project/venv
 ```
 
 This will install an editable version of the package in the test project. Any

pyproject.toml

@@ -28,6 +28,7 @@ splunk_distro = "splunk_otel.distro:_SplunkDistro"
 [tool.poetry.dependencies]
 cryptography=">=2.0,<=41.0.4"
 python = "^3.7.2"
+protobuf = "^4.23"
 opentelemetry-api = "1.20.0"
 opentelemetry-sdk = "1.20.0"
 opentelemetry-instrumentation = "0.41b0"

splunk_otel/__init__.py

@@ -20,10 +20,14 @@
 """
 
 from .defaults import _set_otel_defaults
+from .util import _init_logger
+
+_init_logger("splunk_otel")
 
 _set_otel_defaults()
 
 # pylint: disable=wrong-import-position
 from .metrics import start_metrics  # noqa: E402
+from .profiling import start_profiling  # noqa: E402
 from .tracing import start_tracing  # noqa: E402
 from .version import __version__  # noqa: E402

splunk_otel/distro.py

@@ -21,19 +21,20 @@
 
 from splunk_otel.metrics import _configure_metrics
 from splunk_otel.options import _Options
+from splunk_otel.profiling import _start_profiling
+from splunk_otel.profiling.options import _Options as ProfilingOptions
 from splunk_otel.tracing import _configure_tracing
 from splunk_otel.util import _is_truthy
 
-otel_log_level = os.environ.get("OTEL_LOG_LEVEL", logging.INFO)
-
-logger = logging.getLogger(__file__)
-logger.setLevel(otel_log_level)
+logger = logging.getLogger(__name__)
 
 
 class _SplunkDistro(BaseDistro):
     def __init__(self):
         tracing_enabled = os.environ.get("OTEL_TRACE_ENABLED", True)
+        profiling_enabled = os.environ.get("SPLUNK_PROFILER_ENABLED", False)
         self._tracing_enabled = _is_truthy(tracing_enabled)
+        self._profiling_enabled = _is_truthy(profiling_enabled)
         if not self._tracing_enabled:
             logger.info(
                 "tracing has been disabled with OTEL_TRACE_ENABLED=%s", tracing_enabled
@@ -47,8 +48,14 @@ def __init__(self):
             )
 
     def _configure(self, **kwargs: Dict[str, Any]) -> None:
+        options = _Options()
+
         if self._tracing_enabled:
-            _configure_tracing(_Options())
+            _configure_tracing(options)
+
+        if self._profiling_enabled:
+            _start_profiling(ProfilingOptions(options.resource))
+
         if self._metrics_enabled:
             _configure_metrics()
 

splunk_otel/metrics.py

@@ -23,10 +23,7 @@
 
 from splunk_otel.util import _is_truthy
 
-otel_log_level = os.environ.get("OTEL_LOG_LEVEL", logging.INFO)
-
-logger = logging.getLogger(__file__)
-logger.setLevel(otel_log_level)
+logger = logging.getLogger(__name__)
 
 
 def start_metrics() -> MeterProvider: