From 9a31f8b508ef5a9f3286abf808b714f001ee9071 Mon Sep 17 00:00:00 2001 From: Joe Talerico aka rook Date: Wed, 23 Oct 2024 17:50:42 -0400 Subject: [PATCH] Adding podMem Adding the podMem similar to the podCPU to our csv/result data Signed-off-by: Joe Talerico aka rook --- cmd/k8s-netperf/k8s-netperf.go | 3 +++ pkg/archive/archive.go | 29 ++++++++++++++++++++++++++--- pkg/metrics/system.go | 29 ++++++++++++++++++++++++++++- pkg/results/result.go | 16 ++++++++++++++++ 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go index 44fcbeb..379926f 100644 --- a/cmd/k8s-netperf/k8s-netperf.go +++ b/cmd/k8s-netperf/k8s-netperf.go @@ -259,6 +259,8 @@ var rootCmd = &cobra.Command{ sr.Results[i].ServerMetrics, _ = metrics.QueryNodeCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime) sr.Results[i].ClientPodCPU, _ = metrics.TopPodCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime) sr.Results[i].ServerPodCPU, _ = metrics.TopPodCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime) + sr.Results[i].ClientPodMem, _ = metrics.TopPodMem(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime) + sr.Results[i].ServerPodMem, _ = metrics.TopPodMem(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime) } } } @@ -319,6 +321,7 @@ var rootCmd = &cobra.Command{ if showMetrics { result.ShowNodeCPU(sr) result.ShowPodCPU(sr) + result.ShowPodMem(sr) } } else { err = archive.WriteJSONResult(sr) diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 4fc19d2..d575ecb 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -44,8 +44,10 @@ type Doc struct { Metadata result.Metadata `json:"metadata"` ServerNodeCPU metrics.NodeCPU `json:"serverCPU"` ServerPodCPU []metrics.PodCPU `json:"serverPods"` + ServerPodMem []metrics.PodMem `json:"serverPodsMem"` ClientNodeCPU metrics.NodeCPU `json:"clientCPU"` ClientPodCPU []metrics.PodCPU `json:"clientPods"` + ClientPodMem []metrics.PodMem `json:"clientPodsMem"` Confidence []float64 `json:"confidence"` ServerNodeInfo metrics.NodeInfo `json:"serverNodeInfo"` ClientNodeInfo metrics.NodeInfo `json:"clientNodeInfo"` @@ -108,6 +110,8 @@ func BuildDocs(sr result.ScenarioResults, uuid string) ([]interface{}, error) { ServerNodeCPU: r.ServerMetrics, ClientNodeCPU: r.ClientMetrics, ServerPodCPU: r.ServerPodCPU.Results, + ServerPodMem: r.ServerPodMem.MemResults, + ClientPodMem: r.ClientPodMem.MemResults, ClientPodCPU: r.ClientPodCPU.Results, Metadata: sr.Metadata, AcrossAZ: r.AcrossAZ, @@ -189,7 +193,7 @@ func commonCsvDataFields(row result.Data) []string { } // Writes all the mertics to the archive. -func writeArchive(cpuarchive, podarchive *csv.Writer, role string, row result.Data, podResults []metrics.PodCPU) error { +func writeArchive(cpuarchive, podarchive, podmemarchive *csv.Writer, role string, row result.Data, podResults []metrics.PodCPU, podMem []metrics.PodMem) error { roleFieldData := []string{role} for _, pod := range podResults { if err := podarchive.Write(append(append(roleFieldData, @@ -200,6 +204,15 @@ func writeArchive(cpuarchive, podarchive *csv.Writer, role string, row result.Da return fmt.Errorf("failed to write archive to file") } } + for _, pod := range podMem { + if err := podmemarchive.Write(append(append(roleFieldData, + commonCsvDataFields(row)...), + pod.Name, + fmt.Sprintf("%f", pod.Value), + )); err != nil { + return fmt.Errorf("failed to write archive to file") + } + } cpu := row.ClientMetrics if role == "Server" { @@ -223,6 +236,12 @@ func writeArchive(cpuarchive, podarchive *csv.Writer, role string, row result.Da // WritePromCSVResult writes the prom data in CSV format func WritePromCSVResult(r result.ScenarioResults) error { d := time.Now().Unix() + + podmemfp, err := os.Create(fmt.Sprintf("podmem-result-%d.csv", d)) + if err != nil { + return fmt.Errorf("failed to open pod mem archive file") + } + defer podmemfp.Close() podfp, err := os.Create(fmt.Sprintf("podcpu-result-%d.csv", d)) if err != nil { return fmt.Errorf("failed to open pod cpu archive file") @@ -237,6 +256,9 @@ func WritePromCSVResult(r result.ScenarioResults) error { defer cpuarchive.Flush() podarchive := csv.NewWriter(podfp) defer podarchive.Flush() + + podmemarchive := csv.NewWriter(podmemfp) + defer podmemarchive.Flush() roleField := []string{"Role"} cpudata := append(append(roleField, commonCsvHeaderFields()...), @@ -260,13 +282,14 @@ func WritePromCSVResult(r result.ScenarioResults) error { return fmt.Errorf("failed to write pod archive to file") } for _, row := range r.Results { - if err := writeArchive(cpuarchive, podarchive, "Client", row, row.ClientPodCPU.Results); err != nil { + if err := writeArchive(cpuarchive, podarchive, podmemarchive, "Client", row, row.ClientPodCPU.Results, row.ClientPodMem.MemResults); err != nil { return err } - if err := writeArchive(cpuarchive, podarchive, "Server", row, row.ServerPodCPU.Results); err != nil { + if err := writeArchive(cpuarchive, podarchive, podmemarchive, "Server", row, row.ServerPodCPU.Results, row.ServerPodMem.MemResults); err != nil { return err } } + return nil } diff --git a/pkg/metrics/system.go b/pkg/metrics/system.go index 5c9cda6..f2cb59c 100644 --- a/pkg/metrics/system.go +++ b/pkg/metrics/system.go @@ -40,9 +40,15 @@ type PodCPU struct { Value float64 `json:"cpuUsage"` } +type PodMem struct { + Name string `json:"podName"` + Value float64 `json:"memUsage"` +} + // PodValues is a collection of PodCPU type PodValues struct { - Results []PodCPU + Results []PodCPU + MemResults []PodMem } // PromConnect stores the prom information @@ -187,6 +193,27 @@ func TopPodCPU(node NodeInfo, conn PromConnect, start time.Time, end time.Time) return pods, true } +// TopPodMem will return the top 5 Mem consumers for a specific node +func TopPodMem(node NodeInfo, conn PromConnect, start time.Time, end time.Time) (PodValues, bool) { + var pods PodValues + query := fmt.Sprintf("topk(5,container_memory_rss{container!=\"POD\",name!=\"\",node=~\"%s\"})", node.NodeName) + logging.Debugf("Prom Query : %s", query) + val, err := conn.Client.QueryRange(query, start, end, time.Minute) + if err != nil { + logging.Error("Issue querying Prometheus") + return pods, false + } + v := val.(model.Matrix) + for _, s := range v { + p := PodMem{ + Name: string(s.Metric["pod"]), + Value: avg(s.Values), + } + pods.MemResults = append(pods.MemResults, p) + } + return pods, true +} + // Calculates average for the given data func avg(data []model.SamplePair) float64 { sum := 0.0 diff --git a/pkg/results/result.go b/pkg/results/result.go index 6b701ef..b2a7646 100644 --- a/pkg/results/result.go +++ b/pkg/results/result.go @@ -43,7 +43,9 @@ type Data struct { ClientMetrics metrics.NodeCPU ServerMetrics metrics.NodeCPU ClientPodCPU metrics.PodValues + ClientPodMem metrics.PodValues ServerPodCPU metrics.PodValues + ServerPodMem metrics.PodValues } // ScenarioResults each scenario could have multiple results @@ -201,6 +203,20 @@ func ShowPodCPU(s ScenarioResults) { table.Render() } +// ShowPodMem accepts ScenarioResults and presents to the user via stdout the Podmem info +func ShowPodMem(s ScenarioResults) { + table := initTable([]string{"Result Type", "Driver", "Role", "Scenario", "Parallelism", "Host Network", "Service", "Message Size", "Burst", "Same node", "Pod", "Utilization"}) + for _, r := range s.Results { + for _, pod := range r.ClientPodMem.MemResults { + table.Append([]string{"Pod Mem RSS Utilization", r.Driver, "Client", r.Profile, fmt.Sprintf("%d", r.Parallelism), fmt.Sprintf("%t", r.HostNetwork), fmt.Sprintf("%t", r.Service), fmt.Sprintf("%d", r.MessageSize), fmt.Sprintf("%d", r.Burst), fmt.Sprintf("%t", r.SameNode), fmt.Sprintf("%.20s", pod.Name), fmt.Sprintf("%f", pod.Value)}) + } + for _, pod := range r.ServerPodMem.MemResults { + table.Append([]string{"Pod Mem RSS Utilization", r.Driver, "Server", r.Profile, fmt.Sprintf("%d", r.Parallelism), fmt.Sprintf("%t", r.HostNetwork), fmt.Sprintf("%t", r.Service), fmt.Sprintf("%d", r.MessageSize), fmt.Sprintf("%d", r.Burst), fmt.Sprintf("%t", r.SameNode), fmt.Sprintf("%.20s", pod.Name), fmt.Sprintf("%f", pod.Value)}) + } + } + table.Render() +} + // ShowNodeCPU accepts ScenarioResults and presents to the user via stdout the NodeCPU info func ShowNodeCPU(s ScenarioResults) { table := initTable([]string{"Result Type", "Driver", "Role", "Scenario", "Parallelism", "Host Network", "Service", "Message Size", "Burst", "Same node", "Idle CPU", "User CPU", "System CPU", "Steal CPU", "IOWait CPU", "Nice CPU", "SoftIRQ CPU", "IRQ CPU"})