Support defining ClusterLogForwarding filter list as an object

.cruft.json

@@ -1,13 +1,13 @@
 {
   "template": "https://github.com/projectsyn/commodore-component-template.git",
-  "commit": "8840f87d25d97ce0d4bfed75d40173caaf4100fc",
+  "commit": "0aba6c3dc0f4bcbe81aab1cdcc977f32b00f1cb2",
   "checkout": "main",
   "context": {
     "cookiecutter": {
       "name": "OpenShift4 Logging",
       "slug": "openshift4-logging",
       "parameter_key": "openshift4_logging",
-      "test_cases": "defaults master elasticsearch multilineerr forwardingonly legacy",
+      "test_cases": "defaults master elasticsearch multilineerr forwardingonly legacy log-forwarding",
       "add_lib": "n",
       "add_pp": "n",
       "add_golden": "y",

.github/workflows/test.yaml

@@ -39,6 +39,7 @@ jobs:
           - multilineerr
           - forwardingonly
           - legacy
+          - log-forwarding
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}
@@ -59,6 +60,7 @@ jobs:
           - multilineerr
           - forwardingonly
           - legacy
+          - log-forwarding
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}

Makefile.vars.mk

@@ -57,4 +57,4 @@ KUBENT_IMAGE    ?= ghcr.io/doitintl/kube-no-trouble:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
 instance ?= defaults
-test_instances = tests/defaults.yml tests/master.yml tests/elasticsearch.yml tests/multilineerr.yml tests/forwardingonly.yml tests/legacy.yml
+test_instances = tests/defaults.yml tests/master.yml tests/elasticsearch.yml tests/multilineerr.yml tests/forwardingonly.yml tests/legacy.yml tests/log-forwarding.yml

component/config_forwarding.libsonnet

@@ -202,25 +202,31 @@ local clusterLogForwarderSpec = std.foldl(
 
 // Unfold objects into array for ClusterLogForwarder resource.
 local unfoldSpecs(specs) = {
-  // Unfold objects into array.
-  [if std.length(specs.inputs) > 0 then 'inputs']: [
-    { name: name } + specs.inputs[name]
-    for name in std.objectFields(specs.inputs)
-  ],
-  [if std.length(specs.outputs) > 0 then 'outputs']: [
-    { name: name } + specs.outputs[name]
-    for name in std.objectFields(specs.outputs)
-  ],
-  [if std.length(specs.pipelines) > 0 then 'pipelines']: [
-    { name: name } + specs.pipelines[name]
-    for name in std.objectFields(specs.pipelines)
-  ],
-} + {
-  // Import remaining specs as is.
-  [key]: specs[key]
-  for key in std.objectFields(specs)
-  if !std.member([ 'inputs', 'outputs', 'pipelines' ], key)
-};
+                             // Unfold objects into array.
+                             [if std.length(specs.inputs) > 0 then 'inputs']: [
+                               { name: name } + specs.inputs[name]
+                               for name in std.objectFields(specs.inputs)
+                             ],
+                             [if std.length(specs.outputs) > 0 then 'outputs']: [
+                               { name: name } + specs.outputs[name]
+                               for name in std.objectFields(specs.outputs)
+                             ],
+                             [if std.length(specs.pipelines) > 0 then 'pipelines']: [
+                               { name: name } + specs.pipelines[name]
+                               for name in std.objectFields(specs.pipelines)
+                             ],
+                             [if std.objectHas(specs, 'filters') && std.length(specs.filters) > 0 then 'filters']: (
+                               if std.isObject(specs.filters) then
+                                 [ { name: name } + specs.filters[name] for name in std.objectFields(specs.filters) ]
+                               else specs.filters
+                             ),
+                           }
+                           + {
+                             // Import remaining specs as is.
+                             [key]: specs[key]
+                             for key in std.objectFields(specs)
+                             if !std.member([ 'inputs', 'outputs', 'pipelines', 'filters' ], key)
+                           };
 
 // ClusterLogForwarder:
 // Create definitive ClusterLogForwarder resource from specs.

tests/golden/legacy/openshift4-logging/openshift4-logging/31_cluster_logforwarding.yaml

@@ -7,6 +7,13 @@ metadata:
   name: instance
   namespace: openshift-logging
 spec:
+  filters:
+    - drop:
+        - test:
+            - field: .message
+              matches: .*DEBUG.*
+      name: legacy-filter
+      type: drop
   inputs:
     - application:
         namespaces:

tests/golden/log-forwarding/openshift4-logging/console-patching/openshift4_console_params.yaml

@@ -0,0 +1,3 @@
+config:
+  plugins:
+    - logging-view-plugin

tests/golden/log-forwarding/openshift4-logging/openshift4-logging/00_namespace.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations:
+    openshift.io/node-selector: ''
+  labels:
+    name: openshift-logging
+    openshift.io/cluster-monitoring: 'true'
+  name: openshift-logging

tests/golden/log-forwarding/openshift4-logging/openshift4-logging/10_operator_group.yaml

@@ -0,0 +1,11 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  annotations: {}
+  labels:
+    name: cluster-logging
+  name: cluster-logging
+  namespace: openshift-logging
+spec:
+  targetNamespaces:
+    - openshift-logging

tests/golden/log-forwarding/openshift4-logging/openshift4-logging/20_subscriptions.yaml

@@ -0,0 +1,43 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  annotations: {}
+  labels:
+    name: cluster-logging
+  name: cluster-logging
+  namespace: openshift-logging
+spec:
+  channel: stable-5.9
+  config:
+    resources:
+      limits:
+        memory: 256Mi
+      requests:
+        cpu: 10m
+        memory: 128Mi
+  installPlanApproval: Automatic
+  name: cluster-logging
+  source: redhat-operators
+  sourceNamespace: openshift-operators-redhat
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  annotations: {}
+  labels:
+    name: loki-operator
+  name: loki-operator
+  namespace: openshift-operators-redhat
+spec:
+  channel: stable-5.9
+  config:
+    resources:
+      limits:
+        memory: 512Mi
+      requests:
+        cpu: 50m
+        memory: 381Mi
+  installPlanApproval: Automatic
+  name: loki-operator
+  source: openshift-operators-redhat
+  sourceNamespace: openshift-operators-redhat

tests/golden/log-forwarding/openshift4-logging/openshift4-logging/30_cluster_logging.yaml

@@ -0,0 +1,17 @@
+apiVersion: logging.openshift.io/v1
+kind: ClusterLogging
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+  labels:
+    name: instance
+  name: instance
+  namespace: openshift-logging
+spec:
+  collection:
+    type: vector
+  logStore:
+    lokistack:
+      name: loki
+    type: lokistack
+  managementState: Managed

tests/golden/log-forwarding/openshift4-logging/openshift4-logging/31_cluster_logforwarding.yaml

@@ -0,0 +1,46 @@
+apiVersion: logging.openshift.io/v1
+kind: ClusterLogForwarder
+metadata:
+  annotations: {}
+  labels:
+    name: instance
+  name: instance
+  namespace: openshift-logging
+spec:
+  filters:
+    - drop:
+        - test:
+            - field: .message
+              notMatches: .*org\.keycloak\.events.*
+      name: keycloak-filter
+      type: drop
+  inputs:
+    - application:
+        namespaces:
+          - vshn-keycloak-prod
+      name: keycloak-logs
+  outputs:
+    - loki:
+        tenantKey: keycloak
+      name: appuio-loki-keycloak
+      type: loki
+      url: https://my-loki-url.com
+  pipelines:
+    - inputRefs:
+        - application
+      name: application-logs
+      outputRefs:
+        - default
+    - inputRefs:
+        - infrastructure
+      name: infrastructure-logs
+      outputRefs:
+        - default
+    - filterRefs:
+        - keycloak-filter
+      inputRefs:
+        - keycloak-logs
+      name: keycloak-logs
+      outputRefs:
+        - appuio-loki-keycloak
+        - default

tests/golden/log-forwarding/openshift4-logging/openshift4-logging/50_loki_ingester_fix.yaml

@@ -0,0 +1,153 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations: {}
+  labels:
+    name: loki-ingester-check
+  name: loki-ingester-check
+  namespace: openshift-logging
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  annotations: {}
+  labels:
+    name: loki-ingester-check
+  name: loki-ingester-check
+  namespace: openshift-logging
+rules:
+  - apiGroups:
+      - ''
+    resources:
+      - pods
+      - pods/exec
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - delete
+      - patch
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  annotations: {}
+  labels:
+    name: loki-ingester-check
+  name: loki-ingester-check
+  namespace: openshift-logging
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: loki-ingester-check
+subjects:
+  - kind: ServiceAccount
+    name: loki-ingester-check
+---
+apiVersion: v1
+data:
+  wal-check.sh: |
+    #!/bin/bash
+
+    set -e -o pipefail
+
+    # Check if pod is in stuck state.
+    function check_pod() {
+      POD_NAME="loki-ingester-${1}"
+      echo "checking POD ${POD_NAME}"
+      PHASE=$(kubectl -n openshift-logging get po ${POD_NAME} -oyaml | yq '.status.phase')
+      if [ ${PHASE} != "Running" ]; then
+        return 0
+      fi
+      READY=$(kubectl -n openshift-logging get po ${POD_NAME} -oyaml | yq '.status.conditions[] | select(.type == "ContainersReady") | .status')
+      if [ ${READY} == "True" ]; then
+        return 0
+      fi
+      return 1
+    }
+
+    # Check directories of pod and remove non-existing checkpoint if present.
+    function check_dir() {
+      shopt -s extglob
+      POD_NAME="loki-ingester-${1}"
+      echo "checking DIR ${POD_NAME}"
+      DIR_CHP=$(kubectl -n openshift-logging exec -i ${POD_NAME} -- ls /tmp/wal | grep -o "^checkpoint\.[0-9]*$")
+      PATTERN=$(echo ${DIR_CHP} | sed 's/[^0-9]*//g')
+      DIR_WAL=$(kubectl -n openshift-logging exec -i ${POD_NAME} -- ls /tmp/wal | grep -o "^0*${PATTERN}$" || exit 0)
+      if [ -z $DIR_WAL ]; then
+        kubectl -n openshift-logging exec -i ${POD_NAME} -- rm -rf /tmp/wal/${DIR_CHP}
+        kubectl -n openshift-logging delete po ${POD_NAME}
+      fi
+    }
+
+    # Check if pods are in stuck state for longer than ${SLEEP_TIME}.
+    # Only fix 1 pod at a time and immediatly exit if it is fixed.
+    function fix_pod() {
+      if ! check_pod $1; then
+        echo "stuck POD, waiting ${SLEEP_TIME}"
+        sleep ${SLEEP_TIME}
+        if ! check_pod $1; then
+          check_dir $1
+          exit 0
+        fi
+      fi
+    }
+
+    fix_pod 0
+    fix_pod 1
+
+    exit 0
+kind: ConfigMap
+metadata:
+  annotations: {}
+  labels:
+    name: loki-ingester-check
+  name: loki-ingester-check
+  namespace: openshift-logging
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  annotations: {}
+  labels:
+    name: loki-ingester-check
+  name: loki-ingester-check
+  namespace: openshift-logging
+spec:
+  concurrencyPolicy: Forbid
+  failedJobsHistoryLimit: 0
+  jobTemplate:
+    spec:
+      activeDeadlineSeconds: 360
+      backoffLimit: 1
+      template:
+        spec:
+          containers:
+            - command:
+                - /usr/local/bin/wal-check.sh
+              env:
+                - name: SLEEP_TIME
+                  value: 2m
+              image: quay.io/appuio/oc:v4.14
+              imagePullPolicy: IfNotPresent
+              name: check-pod
+              ports: []
+              stdin: false
+              tty: false
+              volumeMounts:
+                - mountPath: /usr/local/bin/wal-check.sh
+                  name: wal-check
+                  readOnly: true
+                  subPath: wal-check.sh
+          nodeSelector:
+            node-role.kubernetes.io/infra: ''
+          restartPolicy: Never
+          serviceAccountName: loki-ingester-check
+          volumes:
+            - configMap:
+                defaultMode: 364
+                name: loki-ingester-check
+              name: wal-check
+  schedule: '*/10 * * * *'