From 686e6db458d01097bc1d9fd140ac5fc1a530c6c2 Mon Sep 17 00:00:00 2001 From: Syed Nihal Date: Mon, 25 Mar 2024 14:16:40 +0530 Subject: [PATCH 1/3] adding support for env variable expansion in log config. see https://github.com/kiwigrid/k8s-sidecar/issues/337 Signed-off-by: Syed Nihal --- README.md | 29 +++++++++++++++++++++++++++++ src/logger.py | 19 ++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9008c529..8f464d55 100644 --- a/README.md +++ b/README.md @@ -93,3 +93,32 @@ If the filename ends with `.url` suffix, the content will be processed as a URL | `LOG_FORMAT` | Set a log format. (JSON or LOGFMT) | false | `JSON` | string | | `LOG_TZ` | Set the log timezone. (LOCAL or UTC) | false | `LOCAL` | string | | `LOG_CONFIG` | Log configuration file path. If not configured, uses the default log config for backward compatibility support. When not configured `LOG_LEVEL, LOG_FORMAT and LOG_TZ` would be used. Refer to [Python logging](https://docs.python.org/3/library/logging.config.html) for log configuration. For sample configuration file refer to file examples/example_logconfig.yaml | false | - | string | + +# Environment variable expansion in LOG_CONFIG +Kiwigrid k8s-sidecar supports expansion fo environment variables expansion in the log config. +This can be done by wrapping the name of environment variable in the regex placeholder `$()` in the log config. +At the startup, the k8s container will look for the regex wrapper and replace all the matched occurrences with the content of the environment variables. + +For instance the below snippet from the log config, + +```commandline +version: 1 +disable_existing_loggers: false + +root: + level: $(LV_DBG) + handlers: [console] +... +``` + +would be read as below, replacing the content of `$(LV_DBG)` with the value of environment variable `LV_DBG`. + +```commandline +version: 1 +disable_existing_loggers: false + +root: + level: DEBUG + handlers: [console] +... +``` \ No newline at end of file diff --git a/src/logger.py b/src/logger.py index e6324abc..27ef75e2 100644 --- a/src/logger.py +++ b/src/logger.py @@ -1,6 +1,7 @@ import logging import os import sys +import re import yaml from datetime import datetime from typing import Optional @@ -107,11 +108,27 @@ def add_fields(self, log_record, record, message_dict): } } +def expand_env(data): + placeholder_pattern = r'\$\((.*?)\)' + + def replace_placeholder(s): + env = s.group(1) + env_value = os.getenv(env) + if env_value is None: + print(f'unable to expand environment variable {env} in LOG_CONFIG. reason: env variable not set') + sys.exit(1) + else: + return env_value + + processed_data = re.sub(placeholder_pattern, replace_placeholder, data) + return yaml.load(processed_data, Loader=yaml.FullLoader) + def get_log_config(): if log_conf_file != "" : try: with open(log_conf_file, 'r') as stream: - config = yaml.load(stream, Loader=yaml.FullLoader) + data = stream.read() + config = expand_env(data) return config except FileNotFoundError: msg = "Config file: "+ log_conf_file + " Not Found" From a1f421b1545e4622d23e2d5835a723a6ea223cb3 Mon Sep 17 00:00:00 2001 From: Syed Nihal Date: Thu, 4 Apr 2024 20:50:37 +0530 Subject: [PATCH 2/3] made changes in tests for env expansion Signed-off-by: Syed Nihal --- README.md | 2 +- test/resources/sidecar.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8f464d55..d4d52412 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ If the filename ends with `.url` suffix, the content will be processed as a URL | `LOG_CONFIG` | Log configuration file path. If not configured, uses the default log config for backward compatibility support. When not configured `LOG_LEVEL, LOG_FORMAT and LOG_TZ` would be used. Refer to [Python logging](https://docs.python.org/3/library/logging.config.html) for log configuration. For sample configuration file refer to file examples/example_logconfig.yaml | false | - | string | # Environment variable expansion in LOG_CONFIG -Kiwigrid k8s-sidecar supports expansion fo environment variables expansion in the log config. +Kiwigrid k8s-sidecar supports expansion of environment variables in the log config. This can be done by wrapping the name of environment variable in the regex placeholder `$()` in the log config. At the startup, the k8s container will look for the regex wrapper and replace all the matched occurrences with the content of the environment variables. diff --git a/test/resources/sidecar.yaml b/test/resources/sidecar.yaml index 0d76c161..a1094637 100644 --- a/test/resources/sidecar.yaml +++ b/test/resources/sidecar.yaml @@ -204,7 +204,7 @@ data: class: logging.handlers.RotatingFileHandler level: DEBUG formatter: JSON - filename: "/opt/logs/sidecar.log" + filename: "$(OUTPUT_LOG_FILE)" maxBytes: 2097152 backupCount: 3 @@ -294,6 +294,8 @@ spec: value: "DEBUG" - name: LOG_CONFIG value: "/etc/k8s-sidecar/log_conf.yaml" + - name: OUTPUT_LOG_FILE + value: "/opt/logs/sidecar.log" volumes: - name: shared-volume emptyDir: { } From e4ee00b5e902477223258c5400701dac8b15b70a Mon Sep 17 00:00:00 2001 From: wasim-nihal Date: Wed, 25 Sep 2024 16:54:22 +0530 Subject: [PATCH 3/3] Added test scenarios for environment variable expansion in log config Signed-off-by: wasim-nihal --- .github/workflows/build_and_test.yaml | 6 +++++- README.md | 11 +++++++---- test/resources/sidecar.yaml | 18 +++++++++++------- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 993a8ffc..c7912fb2 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -246,4 +246,8 @@ jobs: kubectl exec sidecar -- sh -c "! test -e /tmp/relative/relative.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/relative/change-relative.txt" && kubectl exec sidecar -- sh -c "! test -e /tmp/orig-dir/change-dir.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/new-dir/change-dir.txt" && kubectl exec sidecar -- sh -c "! test -e /tmp/similar-configmap.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-configmap.txt" && - kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt" \ No newline at end of file + kubectl exec sidecar -- sh -c "! test -e /tmp/similar-secret.txt" && kubectl exec sidecar -- sh -c "test -e /tmp/change-similar-secret.txt" + - name: Verify environment variable expansion feature in log config + run: | + cat /tmp/logs/sidecar-pythonscript-logfile.log | jq 'has("msg") and has("level")' -e >/dev/null && + test $(cat /tmp/logs/sidecar-pythonscript-logfile.log | jq '. | select(.level == "DEBUG")' | wc -l) -gt 0 \ No newline at end of file diff --git a/README.md b/README.md index 79c09ec8..4f7fc086 100644 --- a/README.md +++ b/README.md @@ -100,10 +100,11 @@ If the filename ends with `.url` suffix, the content will be processed as a URL | `LOG_TZ` | Set the log timezone. (LOCAL or UTC) | false | `LOCAL` | string | | `LOG_CONFIG` | Log configuration file path. If not configured, uses the default log config for backward compatibility support. When not configured `LOG_LEVEL, LOG_FORMAT and LOG_TZ` would be used. Refer to [Python logging](https://docs.python.org/3/library/logging.config.html) for log configuration. For sample configuration file refer to file examples/example_logconfig.yaml | false | - | string | -# Environment variable expansion in LOG_CONFIG + +## Environment variable expansion in LOG_CONFIG Kiwigrid k8s-sidecar supports expansion of environment variables in the log config. This can be done by wrapping the name of environment variable in the regex placeholder `$()` in the log config. -At the startup, the k8s container will look for the regex wrapper and replace all the matched occurrences with the content of the environment variables. +At the startup, the k8s-sidecar container will look for the regex wrapper and replace all the matched occurrences with the content of the environment variables. For instance the below snippet from the log config, @@ -117,7 +118,7 @@ root: ... ``` -would be read as below, replacing the content of `$(LV_DBG)` with the value of environment variable `LV_DBG`. +would be read as below, replacing the content of `$(LV_DBG)` with the value of environment variable `LV_DBG` (assuming it set to `DEBUG` here). ```commandline version: 1 @@ -127,4 +128,6 @@ root: level: DEBUG handlers: [console] ... -``` \ No newline at end of file +``` + +> **Note:** The k8s-sidecar will terminate if it finds a match for the environment variable expansion placeholder and the corresponding environment variable is not set. diff --git a/test/resources/sidecar.yaml b/test/resources/sidecar.yaml index edda7b54..d4bcc021 100644 --- a/test/resources/sidecar.yaml +++ b/test/resources/sidecar.yaml @@ -166,13 +166,13 @@ data: disable_existing_loggers: false root: - level: DEBUG + level: $(ENV_LEVEL_DEBUG) handlers: [console] handlers: console: class: logging.StreamHandler - level: DEBUG + level: $(ENV_LEVEL_DEBUG) formatter: JSON formatters: @@ -180,8 +180,8 @@ data: (): logger.JsonFormatter format: '%(levelname)s %(message)s' rename_fields: { - "message": "msg", - "levelname": "level" + "message": "$(ENV_RENAME_FIELD_MSG)", + "levelname": "$(ENV_RENAME_FIELD_LEVEL)" } --- apiVersion: v1 @@ -204,7 +204,7 @@ data: class: logging.handlers.RotatingFileHandler level: DEBUG formatter: JSON - filename: "$(OUTPUT_LOG_FILE)" + filename: "/opt/logs/sidecar.log" maxBytes: 2097152 backupCount: 3 @@ -248,6 +248,12 @@ spec: value: "DEBUG" - name: LOG_CONFIG value: "/etc/k8s-sidecar/log_conf.yaml" + - name: ENV_LEVEL_DEBUG + value: "DEBUG" + - name: ENV_RENAME_FIELD_MSG + value: "msg" + - name: ENV_RENAME_FIELD_LEVEL + value: "level" volumes: - name: shared-volume emptyDir: { } @@ -294,8 +300,6 @@ spec: value: "DEBUG" - name: LOG_CONFIG value: "/etc/k8s-sidecar/log_conf.yaml" - - name: OUTPUT_LOG_FILE - value: "/opt/logs/sidecar.log" volumes: - name: shared-volume emptyDir: { }