From 869a55cf02ce433883599a05bf88316e0f672229 Mon Sep 17 00:00:00 2001 From: Matej Kralik Date: Fri, 13 Sep 2024 11:18:24 +0200 Subject: [PATCH] [OSSM-8128] Use comunity prometheus operator (#739) --- pkg/prometheusoperator/install.go | 95 +++++++++++++++ .../yaml/custom-prometheus-operator.yaml | 35 ++++++ .../yaml/prometheus-instance.yaml | 45 +++++++ .../observability/custom_prometheus_test.go | 113 +----------------- pkg/util/prometheus/prometheus_struct.go | 3 +- 5 files changed, 181 insertions(+), 110 deletions(-) create mode 100644 pkg/prometheusoperator/install.go create mode 100644 pkg/prometheusoperator/yaml/custom-prometheus-operator.yaml create mode 100644 pkg/prometheusoperator/yaml/prometheus-instance.yaml diff --git a/pkg/prometheusoperator/install.go b/pkg/prometheusoperator/install.go new file mode 100644 index 00000000..37dbe707 --- /dev/null +++ b/pkg/prometheusoperator/install.go @@ -0,0 +1,95 @@ +// Copyright 2024 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prometheusoperator + +import ( + _ "embed" + "fmt" + + "github.com/maistra/maistra-test-tool/pkg/util/oc" + "github.com/maistra/maistra-test-tool/pkg/util/operator" + "github.com/maistra/maistra-test-tool/pkg/util/pod" + "github.com/maistra/maistra-test-tool/pkg/util/retry" + "github.com/maistra/maistra-test-tool/pkg/util/test" +) + +var ( + //go:embed yaml/custom-prometheus-operator.yaml + prometheusSubscriptionYaml string + + //go:embed yaml/prometheus-instance.yaml + prometheusInstanceYaml string + + customPrometheusNamespace = "custom-prometheus-operator" + + prometheusCsvName = "prometheusoperator" + prometheusOperatorSelector = "app.kubernetes.io/name=prometheus-operator" +) + +func Install(t test.TestHelper) { + oc.CreateNamespace(t, customPrometheusNamespace) + t.Log("Instaling custom prometheus operator...") + operator.CreateOperatorViaOlm(t, customPrometheusNamespace, prometheusCsvName, prometheusSubscriptionYaml, prometheusOperatorSelector, nil) +} + +func Uninstall(t test.TestHelper) { + t.Log("Uninstalling custom prometheus") + oc.DeleteFromTemplate(t, customPrometheusNamespace, prometheusInstanceYaml, nil) + operator.DeleteOperatorViaOlm(t, customPrometheusNamespace, prometheusCsvName, prometheusSubscriptionYaml) + oc.DeleteNamespace(t, customPrometheusNamespace) +} + +func InstalPrometheusInstance(t test.TestHelper, permittedNs ...string) { + oc.ApplyTemplate(t, customPrometheusNamespace, prometheusInstanceYaml, nil) + t.Log("Waiting for custom prometheus to be ready") + oc.DefaultOC.WaitFor(t, customPrometheusNamespace, "Prometheus", "prometheus", "condition=Reconciled") + + for _, permitNs := range permittedNs { + oc.ApplyString(t, permitNs, + ` +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: custom-prometheus-permissions +rules: +- apiGroups: [""] + resources: + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"]`, + fmt.Sprintf(` +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: custom-prometheus-permissions +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: custom-prometheus-permissions +subjects: +- kind: ServiceAccount + name: prometheus-k8s + namespace: %s`, customPrometheusNamespace)) + } + retry.UntilSuccess(t, func(t test.TestHelper) { + prometheusPod := pod.MatchingSelector("app.kubernetes.io/name=prometheus-operator", customPrometheusNamespace) + oc.WaitPodRunning(t, prometheusPod) + }) +} diff --git a/pkg/prometheusoperator/yaml/custom-prometheus-operator.yaml b/pkg/prometheusoperator/yaml/custom-prometheus-operator.yaml new file mode 100644 index 00000000..036f77f9 --- /dev/null +++ b/pkg/prometheusoperator/yaml/custom-prometheus-operator.yaml @@ -0,0 +1,35 @@ +# Copyright 2024 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: custom-prometheus-operator + namespace: custom-prometheus-operator +spec: + targetNamespaces: + - custom-prometheus-operator + upgradeStrategy: Default +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: prometheus + namespace: custom-prometheus-operator +spec: + channel: beta + installPlanApproval: Automatic + name: prometheus + source: community-operators + sourceNamespace: openshift-marketplace diff --git a/pkg/prometheusoperator/yaml/prometheus-instance.yaml b/pkg/prometheusoperator/yaml/prometheus-instance.yaml new file mode 100644 index 00000000..4b94049e --- /dev/null +++ b/pkg/prometheusoperator/yaml/prometheus-instance.yaml @@ -0,0 +1,45 @@ +# Copyright 2024 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: prometheus +spec: + securityContext: {} + serviceAccountName: prometheus-k8s + podMonitorSelector: {} + podMonitorNamespaceSelector: + matchLabels: + kubernetes.io/metadata.name: custom-prometheus-operator + serviceMonitorSelector: {} + serviceMonitorNamespaceSelector: + matchLabels: + kubernetes.io/metadata.name: custom-prometheus-operator + podMetadata: + annotations: + sidecar.istio.io/inject: "true" + traffic.sidecar.istio.io/includeInboundPorts: "" + traffic.sidecar.istio.io/includeOutboundIPRanges: "" + proxy.istio.io/config: | + proxyMetadata: + OUTPUT_CERTS: /etc/istio-output-certs + sidecar.istio.io/userVolumeMount: '[{"name": "istio-certs", "mountPath": "/etc/istio-output-certs"}]' + volumes: + - name: istio-certs + emptyDir: + medium: Memory + volumeMounts: + - mountPath: /etc/prom-certs/ + name: istio-certs diff --git a/pkg/tests/tasks/observability/custom_prometheus_test.go b/pkg/tests/tasks/observability/custom_prometheus_test.go index 6454f71d..ff19de8a 100644 --- a/pkg/tests/tasks/observability/custom_prometheus_test.go +++ b/pkg/tests/tasks/observability/custom_prometheus_test.go @@ -21,12 +21,12 @@ import ( "testing" "github.com/maistra/maistra-test-tool/pkg/app" + "github.com/maistra/maistra-test-tool/pkg/prometheusoperator" "github.com/maistra/maistra-test-tool/pkg/util/check/assert" "github.com/maistra/maistra-test-tool/pkg/util/curl" "github.com/maistra/maistra-test-tool/pkg/util/env" "github.com/maistra/maistra-test-tool/pkg/util/ns" "github.com/maistra/maistra-test-tool/pkg/util/oc" - "github.com/maistra/maistra-test-tool/pkg/util/operator" "github.com/maistra/maistra-test-tool/pkg/util/pod" "github.com/maistra/maistra-test-tool/pkg/util/prometheus" "github.com/maistra/maistra-test-tool/pkg/util/retry" @@ -35,7 +35,7 @@ import ( ) func TestCustomPrometheus(t *testing.T) { - const customPrometheusNs = "custom-prometheus" + const customPrometheusNs = "custom-prometheus-operator" test.NewTest(t).Id("custom-prometheus").Groups(test.Full, test.ARM).Run(func(t test.TestHelper) { smcpVer := env.GetSMCPVersion() @@ -59,18 +59,13 @@ func TestCustomPrometheus(t *testing.T) { }) t.LogStep("Installing Prometheus operator") - oc.CreateNamespace(t, customPrometheusNs) - installPrometheusOperator(t, customPrometheusNs) + prometheusoperator.Install(t) t.LogStep("Creating SMCP with Prometheus extension provider") createSmcpWithPrometheusExtensionProvider(t, meshNamespace, customPrometheusNs, ns.Bookinfo) t.LogStep("Installing custom Prometheus") - installPrometheus(t, customPrometheusNs, meshNamespace, ns.Bookinfo) - retry.UntilSuccess(t, func(t test.TestHelper) { - prometheusPod := pod.MatchingSelector("app.kubernetes.io/name=prometheus-operator", customPrometheusNs) - oc.WaitPodRunning(t, prometheusPod) - }) + prometheusoperator.InstalPrometheusInstance(t, meshNamespace, ns.Bookinfo) t.LogStep("Intalling Bookinfo app") oc.WaitSMCPReady(t, meshNamespace, "basic") @@ -86,9 +81,6 @@ func TestCustomPrometheus(t *testing.T) { enableAppMtlsMonitoring(t, customPrometheusNs, ns.Bookinfo) t.LogStep("Waiting for installs to complete") - fullCsvName := operator.GetFullCsvName(t, customPrometheusNs, "rhods-prometheus") - operator.WaitForOperatorInNamespaceReady(t, customPrometheusNs, "k8s-app=prometheus-operator", fullCsvName) - oc.WaitPodReady(t, pod.MatchingSelector("prometheus=prometheus", customPrometheusNs)) bookinfoApp.WaitReady(t) t.LogStep("Sending request to Bookinfo app") @@ -131,30 +123,6 @@ func ocWaitJsonpath(t test.TestHelper, ns, kind, name, jsonpath, expected, succe }) } -func installPrometheusOperator(t test.TestHelper, ns string) { - t.T().Helper() - oc.ApplyString(t, ns, - fmt.Sprintf(` -apiVersion: operators.coreos.com/v1 -kind: OperatorGroup -metadata: - name: custom-prometheus-operators -spec: - targetNamespaces: - - %s`, - ns), - ` -apiVersion: operators.coreos.com/v1alpha1 -kind: Subscription -metadata: - name: rhods-prometheus-operator -spec: - channel: beta - name: rhods-prometheus-operator - source: redhat-operators - sourceNamespace: openshift-marketplace`) -} - func createSmcpWithPrometheusExtensionProvider(t test.TestHelper, smcpNs, prometheusNs, additionalSmmrNs string) { t.T().Helper() oc.ApplyTemplate(t, smcpNs, ` @@ -205,79 +173,6 @@ spec: additionalSmmrNs)) } -func installPrometheus(t test.TestHelper, ns string, permittedNs ...string) { - t.T().Helper() - oc.ApplyString(t, ns, - fmt.Sprintf(` -apiVersion: monitoring.coreos.com/v1 -kind: Prometheus -metadata: - name: prometheus -spec: - securityContext: {} - serviceAccountName: prometheus-k8s - podMonitorSelector: {} - podMonitorNamespaceSelector: - matchLabels: - kubernetes.io/metadata.name: %s - serviceMonitorSelector: {} - serviceMonitorNamespaceSelector: - matchLabels: - kubernetes.io/metadata.name: %s - podMetadata: - annotations: - sidecar.istio.io/inject: "true" - traffic.sidecar.istio.io/includeInboundPorts: "" - traffic.sidecar.istio.io/includeOutboundIPRanges: "" - proxy.istio.io/config: | - proxyMetadata: - OUTPUT_CERTS: /etc/istio-output-certs - sidecar.istio.io/userVolumeMount: '[{"name": "istio-certs", "mountPath": "/etc/istio-output-certs"}]' - volumes: - - name: istio-certs - emptyDir: - medium: Memory - volumeMounts: - - mountPath: /etc/prom-certs/ - name: istio-certs`, - ns, - ns)) - - for _, permitNs := range permittedNs { - oc.ApplyString(t, permitNs, - ` -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: custom-prometheus-permissions -rules: -- apiGroups: [""] - resources: - - services - - endpoints - - pods - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"]`, - fmt.Sprintf(` -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: custom-prometheus-permissions -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: custom-prometheus-permissions -subjects: -- kind: ServiceAccount - name: prometheus-k8s - namespace: %s`, - ns)) - } -} - func enablePrometheusTelemetry(t test.TestHelper, smcpNs string) { t.T().Helper() oc.ApplyString(t, smcpNs, ` diff --git a/pkg/util/prometheus/prometheus_struct.go b/pkg/util/prometheus/prometheus_struct.go index 172d6448..ad53fcd4 100644 --- a/pkg/util/prometheus/prometheus_struct.go +++ b/pkg/util/prometheus/prometheus_struct.go @@ -60,7 +60,8 @@ func (pi *prometheus_struct) Query(t test.TestHelper, ns string, query string) P output := oc.Exec(t, pod.MatchingSelectorFirst(pi.selector, ns), pi.containerName, - fmt.Sprintf("curl -sS -X GET '%s'", urlShellEscaped)) + // comunity prometheus image doesn't have `curl`, use wget instead + fmt.Sprintf("wget -qO- '%s'", urlShellEscaped)) return parsePrometheusResponse(t, output) }