diff --git a/pkg/tests/tasks/observability/custom_prometheus_test.go b/pkg/tests/tasks/observability/custom_prometheus_test.go index dead4ab9..ee969c2a 100644 --- a/pkg/tests/tasks/observability/custom_prometheus_test.go +++ b/pkg/tests/tasks/observability/custom_prometheus_test.go @@ -74,19 +74,29 @@ func TestCustomPrometheus(t *testing.T) { enableIstioProxiesMonitoring(t, customPrometheusNs, meshNamespace, ns.Bookinfo) enableAppMtlsMonitoring(t, customPrometheusNs, ns.Bookinfo) + t.LogStep("Testing if telemetry was enabled") + ocWaitJsonpath(t, meshNamespace, "smcp", "basic", + "{.status.appliedValues.istio.telemetry.enabled}", "true", + "Telemetry was enabled.", "Telemetry failed to enable.") + t.LogStep("Waiting for installs to complete") bookinfoApp.WaitReady(t) + t.LogStep("Wait until istio targets appear in the Prometheus") + retry.UntilSuccess(t, func(t test.TestHelper) { + resp := prometheus.CustomPrometheusTargets(t, customPrometheusNs) + if !strings.Contains(resp, "istiod-monitor") || + !strings.Contains(resp, "app-metrics-monitor-mtls") || + !strings.Contains(resp, "istio-proxies-monitor") { + t.Error("Istio Prometheus targets istiod-monitor/app-metrics-monitor-mtls/istio-proxies-monitor are not ready") + } + }) + t.LogStep("Sending request to Bookinfo app") retry.UntilSuccess(t, func(t test.TestHelper) { curl.Request(t, app.BookinfoProductPageURL(t, meshNamespace), nil, assert.ResponseStatus(http.StatusOK)) }) - t.LogStep("Testing if telemetry was enabled") - ocWaitJsonpath(t, meshNamespace, "smcp", "basic", - "{.status.appliedValues.istio.telemetry.enabled}", "true", - "Telemetry was enabled.", "Telemetry failed to enable.") - t.LogStep("Testing if 'istio_requests_total' metric is available through Prometheus API") retry.UntilSuccess(t, func(t test.TestHelper) { resp := prometheus.CustomPrometheusQuery(t, customPrometheusNs, diff --git a/pkg/tests/tasks/observability/federated_openshift_monitoring_test.go b/pkg/tests/tasks/observability/federated_openshift_monitoring_test.go index ee8e1a67..25739b38 100644 --- a/pkg/tests/tasks/observability/federated_openshift_monitoring_test.go +++ b/pkg/tests/tasks/observability/federated_openshift_monitoring_test.go @@ -17,6 +17,7 @@ package observability import ( "fmt" "net/http" + "strings" "testing" "github.com/maistra/maistra-test-tool/pkg/app" @@ -62,6 +63,17 @@ func TestFederatedOpenShiftMonitoring(t *testing.T) { oc.WaitPodsExist(t, ns.Foo) oc.WaitAllPodsReady(t, ns.Foo) + t.LogStep("Apply federated metrics monitor") + oc.ApplyString(t, meshNamespace, federatedMonitor) + + t.LogStep("Wait until istio targets appear in the Prometheus") + retry.UntilSuccess(t, func(t test.TestHelper) { + resp := prometheus.ThanosTargets(t, monitoringNs) + if !strings.Contains(resp, "serviceMonitor/istio-system/istio-federation") { + t.Error("Istio Prometheus target serviceMonitor/istio-system/istio-federation are not ready") + } + }) + t.LogStep("Generate some ingress traffic") oc.ApplyFile(t, ns.Foo, "https://raw.githubusercontent.com/maistra/istio/maistra-2.6/samples/httpbin/httpbin-gateway.yaml") httpbinURL := fmt.Sprintf("http://%s/headers", istio.GetIngressGatewayHost(t, meshNamespace)) @@ -69,9 +81,6 @@ func TestFederatedOpenShiftMonitoring(t *testing.T) { curl.Request(t, httpbinURL, nil, assert.ResponseStatus(http.StatusOK)) }) - t.LogStep("Apply federated metrics monitor") - oc.ApplyString(t, meshNamespace, federatedMonitor) - t.LogStep("Check istiod metrics") retry.UntilSuccess(t, func(t test.TestHelper) { resp := prometheus.ThanosQuery(t, monitoringNs, `pilot_info{mesh_id="unique-mesh-id"}`) diff --git a/pkg/util/prometheus/prometheus.go b/pkg/util/prometheus/prometheus.go index 2e719d1e..0abbfdff 100644 --- a/pkg/util/prometheus/prometheus.go +++ b/pkg/util/prometheus/prometheus.go @@ -46,6 +46,7 @@ type Prometheus interface { WithSelector(selector string) Prometheus WithContainerName(containerName string) Prometheus Query(t test.TestHelper, ns string, query string) PrometheusResponse + Targets(t test.TestHelper, ns string) string } func Query(t test.TestHelper, ns string, query string) PrometheusResponse { @@ -56,6 +57,14 @@ func CustomPrometheusQuery(t test.TestHelper, ns string, query string) Prometheu return DefaultCustomPrometheus.Query(t, ns, query) } +func CustomPrometheusTargets(t test.TestHelper, ns string) string { + return DefaultCustomPrometheus.Targets(t, ns) +} + func ThanosQuery(t test.TestHelper, ns string, query string) PrometheusResponse { return DefaultThanos.Query(t, ns, query) } + +func ThanosTargets(t test.TestHelper, ns string) string { + return DefaultThanos.Targets(t, ns) +} diff --git a/pkg/util/prometheus/prometheus_struct.go b/pkg/util/prometheus/prometheus_struct.go index ad53fcd4..ae8cc521 100644 --- a/pkg/util/prometheus/prometheus_struct.go +++ b/pkg/util/prometheus/prometheus_struct.go @@ -55,17 +55,15 @@ func (pi *prometheus_struct) WithContainerName(containerName string) Prometheus func (pi *prometheus_struct) Query(t test.TestHelper, ns string, query string) PrometheusResponse { queryString := url.Values{"query": []string{query}}.Encode() - url := fmt.Sprintf(`http://localhost:9090/api/v1/query?%s`, queryString) - urlShellEscaped := strings.ReplaceAll(url, `'`, `'\\''`) - - output := oc.Exec(t, - pod.MatchingSelectorFirst(pi.selector, ns), pi.containerName, - // comunity prometheus image doesn't have `curl`, use wget instead - fmt.Sprintf("wget -qO- '%s'", urlShellEscaped)) - + output := getPrometheusApi(t, pi, ns, fmt.Sprintf(`query?%s`, queryString)) return parsePrometheusResponse(t, output) } +func (pi *prometheus_struct) Targets(t test.TestHelper, ns string) string { + output := getPrometheusApi(t, pi, ns, `targets?state=active`) + return output +} + func parsePrometheusResponse(t test.TestHelper, response string) PrometheusResponse { result := &PrometheusResponse{} err := json.Unmarshal([]byte(response), result) @@ -76,3 +74,14 @@ func parsePrometheusResponse(t test.TestHelper, response string) PrometheusRespo return *result } + +func getPrometheusApi(t test.TestHelper, pi *prometheus_struct, ns string, endpoint string) string { + url := fmt.Sprintf(`http://localhost:9090/api/v1/%s`, endpoint) + urlShellEscaped := strings.ReplaceAll(url, `'`, `'\\''`) + + output := oc.Exec(t, + pod.MatchingSelectorFirst(pi.selector, ns), pi.containerName, + // comunity prometheus image doesn't have `curl`, use wget instead + fmt.Sprintf("wget -qO- '%s'", urlShellEscaped)) + return output +}