Skip to content

Commit

Permalink
[OSSM-8128] Use comunity prometheus operator (#739)
Browse files Browse the repository at this point in the history
  • Loading branch information
mkralik3 authored Sep 13, 2024
1 parent 5f9e6fa commit 869a55c
Show file tree
Hide file tree
Showing 5 changed files with 181 additions and 110 deletions.
95 changes: 95 additions & 0 deletions pkg/prometheusoperator/install.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
// Copyright 2024 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheusoperator

import (
_ "embed"
"fmt"

"github.com/maistra/maistra-test-tool/pkg/util/oc"
"github.com/maistra/maistra-test-tool/pkg/util/operator"
"github.com/maistra/maistra-test-tool/pkg/util/pod"
"github.com/maistra/maistra-test-tool/pkg/util/retry"
"github.com/maistra/maistra-test-tool/pkg/util/test"
)

var (
//go:embed yaml/custom-prometheus-operator.yaml
prometheusSubscriptionYaml string

//go:embed yaml/prometheus-instance.yaml
prometheusInstanceYaml string

customPrometheusNamespace = "custom-prometheus-operator"

prometheusCsvName = "prometheusoperator"
prometheusOperatorSelector = "app.kubernetes.io/name=prometheus-operator"
)

func Install(t test.TestHelper) {
oc.CreateNamespace(t, customPrometheusNamespace)
t.Log("Instaling custom prometheus operator...")
operator.CreateOperatorViaOlm(t, customPrometheusNamespace, prometheusCsvName, prometheusSubscriptionYaml, prometheusOperatorSelector, nil)
}

func Uninstall(t test.TestHelper) {
t.Log("Uninstalling custom prometheus")
oc.DeleteFromTemplate(t, customPrometheusNamespace, prometheusInstanceYaml, nil)
operator.DeleteOperatorViaOlm(t, customPrometheusNamespace, prometheusCsvName, prometheusSubscriptionYaml)
oc.DeleteNamespace(t, customPrometheusNamespace)
}

func InstalPrometheusInstance(t test.TestHelper, permittedNs ...string) {
oc.ApplyTemplate(t, customPrometheusNamespace, prometheusInstanceYaml, nil)
t.Log("Waiting for custom prometheus to be ready")
oc.DefaultOC.WaitFor(t, customPrometheusNamespace, "Prometheus", "prometheus", "condition=Reconciled")

for _, permitNs := range permittedNs {
oc.ApplyString(t, permitNs,
`
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: custom-prometheus-permissions
rules:
- apiGroups: [""]
resources:
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]`,
fmt.Sprintf(`
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: custom-prometheus-permissions
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: custom-prometheus-permissions
subjects:
- kind: ServiceAccount
name: prometheus-k8s
namespace: %s`, customPrometheusNamespace))
}
retry.UntilSuccess(t, func(t test.TestHelper) {
prometheusPod := pod.MatchingSelector("app.kubernetes.io/name=prometheus-operator", customPrometheusNamespace)
oc.WaitPodRunning(t, prometheusPod)
})
}
35 changes: 35 additions & 0 deletions pkg/prometheusoperator/yaml/custom-prometheus-operator.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
# Copyright 2024 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: operators.coreos.com/v1
kind: OperatorGroup
metadata:
name: custom-prometheus-operator
namespace: custom-prometheus-operator
spec:
targetNamespaces:
- custom-prometheus-operator
upgradeStrategy: Default
---
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
name: prometheus
namespace: custom-prometheus-operator
spec:
channel: beta
installPlanApproval: Automatic
name: prometheus
source: community-operators
sourceNamespace: openshift-marketplace
45 changes: 45 additions & 0 deletions pkg/prometheusoperator/yaml/prometheus-instance.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Copyright 2024 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: prometheus
spec:
securityContext: {}
serviceAccountName: prometheus-k8s
podMonitorSelector: {}
podMonitorNamespaceSelector:
matchLabels:
kubernetes.io/metadata.name: custom-prometheus-operator
serviceMonitorSelector: {}
serviceMonitorNamespaceSelector:
matchLabels:
kubernetes.io/metadata.name: custom-prometheus-operator
podMetadata:
annotations:
sidecar.istio.io/inject: "true"
traffic.sidecar.istio.io/includeInboundPorts: ""
traffic.sidecar.istio.io/includeOutboundIPRanges: ""
proxy.istio.io/config: |
proxyMetadata:
OUTPUT_CERTS: /etc/istio-output-certs
sidecar.istio.io/userVolumeMount: '[{"name": "istio-certs", "mountPath": "/etc/istio-output-certs"}]'
volumes:
- name: istio-certs
emptyDir:
medium: Memory
volumeMounts:
- mountPath: /etc/prom-certs/
name: istio-certs
113 changes: 4 additions & 109 deletions pkg/tests/tasks/observability/custom_prometheus_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@ import (
"testing"

"github.com/maistra/maistra-test-tool/pkg/app"
"github.com/maistra/maistra-test-tool/pkg/prometheusoperator"
"github.com/maistra/maistra-test-tool/pkg/util/check/assert"
"github.com/maistra/maistra-test-tool/pkg/util/curl"
"github.com/maistra/maistra-test-tool/pkg/util/env"
"github.com/maistra/maistra-test-tool/pkg/util/ns"
"github.com/maistra/maistra-test-tool/pkg/util/oc"
"github.com/maistra/maistra-test-tool/pkg/util/operator"
"github.com/maistra/maistra-test-tool/pkg/util/pod"
"github.com/maistra/maistra-test-tool/pkg/util/prometheus"
"github.com/maistra/maistra-test-tool/pkg/util/retry"
Expand All @@ -35,7 +35,7 @@ import (
)

func TestCustomPrometheus(t *testing.T) {
const customPrometheusNs = "custom-prometheus"
const customPrometheusNs = "custom-prometheus-operator"

test.NewTest(t).Id("custom-prometheus").Groups(test.Full, test.ARM).Run(func(t test.TestHelper) {
smcpVer := env.GetSMCPVersion()
Expand All @@ -59,18 +59,13 @@ func TestCustomPrometheus(t *testing.T) {
})

t.LogStep("Installing Prometheus operator")
oc.CreateNamespace(t, customPrometheusNs)
installPrometheusOperator(t, customPrometheusNs)
prometheusoperator.Install(t)

t.LogStep("Creating SMCP with Prometheus extension provider")
createSmcpWithPrometheusExtensionProvider(t, meshNamespace, customPrometheusNs, ns.Bookinfo)

t.LogStep("Installing custom Prometheus")
installPrometheus(t, customPrometheusNs, meshNamespace, ns.Bookinfo)
retry.UntilSuccess(t, func(t test.TestHelper) {
prometheusPod := pod.MatchingSelector("app.kubernetes.io/name=prometheus-operator", customPrometheusNs)
oc.WaitPodRunning(t, prometheusPod)
})
prometheusoperator.InstalPrometheusInstance(t, meshNamespace, ns.Bookinfo)

t.LogStep("Intalling Bookinfo app")
oc.WaitSMCPReady(t, meshNamespace, "basic")
Expand All @@ -86,9 +81,6 @@ func TestCustomPrometheus(t *testing.T) {
enableAppMtlsMonitoring(t, customPrometheusNs, ns.Bookinfo)

t.LogStep("Waiting for installs to complete")
fullCsvName := operator.GetFullCsvName(t, customPrometheusNs, "rhods-prometheus")
operator.WaitForOperatorInNamespaceReady(t, customPrometheusNs, "k8s-app=prometheus-operator", fullCsvName)
oc.WaitPodReady(t, pod.MatchingSelector("prometheus=prometheus", customPrometheusNs))
bookinfoApp.WaitReady(t)

t.LogStep("Sending request to Bookinfo app")
Expand Down Expand Up @@ -131,30 +123,6 @@ func ocWaitJsonpath(t test.TestHelper, ns, kind, name, jsonpath, expected, succe
})
}

func installPrometheusOperator(t test.TestHelper, ns string) {
t.T().Helper()
oc.ApplyString(t, ns,
fmt.Sprintf(`
apiVersion: operators.coreos.com/v1
kind: OperatorGroup
metadata:
name: custom-prometheus-operators
spec:
targetNamespaces:
- %s`,
ns),
`
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
name: rhods-prometheus-operator
spec:
channel: beta
name: rhods-prometheus-operator
source: redhat-operators
sourceNamespace: openshift-marketplace`)
}

func createSmcpWithPrometheusExtensionProvider(t test.TestHelper, smcpNs, prometheusNs, additionalSmmrNs string) {
t.T().Helper()
oc.ApplyTemplate(t, smcpNs, `
Expand Down Expand Up @@ -205,79 +173,6 @@ spec:
additionalSmmrNs))
}

func installPrometheus(t test.TestHelper, ns string, permittedNs ...string) {
t.T().Helper()
oc.ApplyString(t, ns,
fmt.Sprintf(`
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
name: prometheus
spec:
securityContext: {}
serviceAccountName: prometheus-k8s
podMonitorSelector: {}
podMonitorNamespaceSelector:
matchLabels:
kubernetes.io/metadata.name: %s
serviceMonitorSelector: {}
serviceMonitorNamespaceSelector:
matchLabels:
kubernetes.io/metadata.name: %s
podMetadata:
annotations:
sidecar.istio.io/inject: "true"
traffic.sidecar.istio.io/includeInboundPorts: ""
traffic.sidecar.istio.io/includeOutboundIPRanges: ""
proxy.istio.io/config: |
proxyMetadata:
OUTPUT_CERTS: /etc/istio-output-certs
sidecar.istio.io/userVolumeMount: '[{"name": "istio-certs", "mountPath": "/etc/istio-output-certs"}]'
volumes:
- name: istio-certs
emptyDir:
medium: Memory
volumeMounts:
- mountPath: /etc/prom-certs/
name: istio-certs`,
ns,
ns))

for _, permitNs := range permittedNs {
oc.ApplyString(t, permitNs,
`
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: custom-prometheus-permissions
rules:
- apiGroups: [""]
resources:
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources:
- configmaps
verbs: ["get"]`,
fmt.Sprintf(`
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: custom-prometheus-permissions
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: custom-prometheus-permissions
subjects:
- kind: ServiceAccount
name: prometheus-k8s
namespace: %s`,
ns))
}
}

func enablePrometheusTelemetry(t test.TestHelper, smcpNs string) {
t.T().Helper()
oc.ApplyString(t, smcpNs, `
Expand Down
3 changes: 2 additions & 1 deletion pkg/util/prometheus/prometheus_struct.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ func (pi *prometheus_struct) Query(t test.TestHelper, ns string, query string) P

output := oc.Exec(t,
pod.MatchingSelectorFirst(pi.selector, ns), pi.containerName,
fmt.Sprintf("curl -sS -X GET '%s'", urlShellEscaped))
// comunity prometheus image doesn't have `curl`, use wget instead
fmt.Sprintf("wget -qO- '%s'", urlShellEscaped))

return parsePrometheusResponse(t, output)
}
Expand Down

0 comments on commit 869a55c

Please sign in to comment.