Virt

README.md

@@ -35,7 +35,7 @@ $ cd k8s-netperf
 $ make docker-build
 ```
 
-## Running
+## Running with Pods
 Ensure your `kubeconfig` is properly set to the cluster you would like to run `k8s-netperf` against.
 
 also be sure to create a `netperf` namespace. (Not over-writable yet)
@@ -95,6 +95,17 @@ Flags:
 
 > *Note: With OpenShift, we attempt to discover the OpenShift route. If that route is not reachable, it might be required to `port-forward` the service and pass that via the `--prom` option.*
 
+## Running with VMs
+Running k8s-netperf against Virtual Machines (OpenShift CNV) requires
+
+- OpenShift CNV must be deployed and users should be able to define VMIs
+- SSH keys to be present in the home directory `(~/.ssh/id_rsa.pub)`
+- OpenShift Routes - k8s-netperf uses this to reach the VMs (k8s-netperf will create the route for the user, but we need Routes)
+
+If the two above are in place, users can orhestrate k8s-netperf to launch VMs by running
+
+`k8s-netperf --vm`
+
 ### Config file
 #### Config File v2
 The v2 config file will be executed in the order the tests are presented in the config file.

cmd/k8s-netperf/k8s-netperf.go

@@ -16,13 +16,15 @@ import (
 	"github.com/cloud-bulldozer/k8s-netperf/pkg/config"
 	"github.com/cloud-bulldozer/k8s-netperf/pkg/drivers"
 	"github.com/cloud-bulldozer/k8s-netperf/pkg/k8s"
+	kubevirtv1 "github.com/cloud-bulldozer/k8s-netperf/pkg/kubevirt/client-go/clientset/versioned/typed/core/v1"
 	log "github.com/cloud-bulldozer/k8s-netperf/pkg/logging"
 	"github.com/cloud-bulldozer/k8s-netperf/pkg/metrics"
 	result "github.com/cloud-bulldozer/k8s-netperf/pkg/results"
 	"github.com/cloud-bulldozer/k8s-netperf/pkg/sample"
 	"github.com/google/uuid"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 )
@@ -39,6 +41,7 @@ var (
 	uperf       bool
 	acrossAZ    bool
 	full        bool
+	vm          bool
 	debug       bool
 	promURL     string
 	id          string
@@ -48,6 +51,7 @@ var (
 	json        bool
 	version     bool
 	csvArchive  bool
+	searchIndex string
 )
 
 var rootCmd = &cobra.Command{
@@ -147,6 +151,21 @@ var rootCmd = &cobra.Command{
 			os.Exit(1)
 		}
 
+		if vm {
+			s.VM = true
+			// Create a dynamic client
+			dynClient, err := dynamic.NewForConfig(rconfig)
+			if err != nil {
+				log.Error(err)
+			}
+			kclient, err := kubevirtv1.NewForConfig(rconfig)
+			if err != nil {
+				log.Error(err)
+			}
+			s.KClient = kclient
+			s.DClient = dynClient
+		}
+
 		// Build the SUT (Deployments)
 		err = k8s.BuildSUT(client, &s)
 		if err != nil {
@@ -174,37 +193,73 @@ var rootCmd = &cobra.Command{
 		if iperf3 {
 			requestedDrivers = append(requestedDrivers, "iperf3")
 		}
+
 		// Run through each test
-		for _, nc := range s.Configs {
-			// Determine the metric for the test
-			metric := string("OP/s")
-			if strings.Contains(nc.Profile, "STREAM") {
-				metric = "Mb/s"
-			}
-			nc.Metric = metric
-			nc.AcrossAZ = acrossAZ
-			// No need to run hostNetwork through Service.
-			var pr result.Data
-			for _, driver := range requestedDrivers {
-				if s.HostNetwork && !nc.Service {
-					pr = executeWorkload(nc, s, true, driver)
+		if !s.VM {
+			for _, nc := range s.Configs {
+				// Determine the metric for the test
+				metric := string("OP/s")
+				if strings.Contains(nc.Profile, "STREAM") {
+					metric = "Mb/s"
+				}
+				nc.Metric = metric
+				nc.AcrossAZ = acrossAZ
+				// No need to run hostNetwork through Service.
+				var pr result.Data
+				for _, driver := range requestedDrivers {
+					if s.HostNetwork && !nc.Service {
+						pr = executeWorkload(nc, s, true, driver, false)
+						if len(pr.Profile) > 1 {
+							sr.Results = append(sr.Results, pr)
+						}
+					}
+					pr = executeWorkload(nc, s, false, driver, false)
 					if len(pr.Profile) > 1 {
 						sr.Results = append(sr.Results, pr)
 					}
 				}
-				pr = executeWorkload(nc, s, false, driver)
-				if len(pr.Profile) > 1 {
-					sr.Results = append(sr.Results, pr)
+			}
+		} else {
+			sr.Virt = true
+			log.Info("Connecting via ssh to the VMI")
+			client, err := k8s.SSHConnect(&s)
+			if err != nil {
+				log.Fatal(err)
+			}
+			s.SSHClient = client
+			for _, nc := range s.Configs {
+				// Determine the metric for the test
+				metric := string("OP/s")
+				if strings.Contains(nc.Profile, "STREAM") {
+					metric = "Mb/s"
+				}
+				nc.Metric = metric
+				nc.AcrossAZ = acrossAZ
+				// No need to run hostNetwork through Service.
+				var pr result.Data
+				for _, driver := range requestedDrivers {
+					if s.HostNetwork && !nc.Service {
+						pr = executeWorkload(nc, s, true, driver, true)
+						if len(pr.Profile) > 1 {
+							sr.Results = append(sr.Results, pr)
+						}
+					}
+					pr = executeWorkload(nc, s, false, driver, true)
+					if len(pr.Profile) > 1 {
+						sr.Results = append(sr.Results, pr)
+					}
 				}
 			}
 		}
 
 		if pavail {
 			for i, npr := range sr.Results {
-				sr.Results[i].ClientMetrics, _ = metrics.QueryNodeCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime)
-				sr.Results[i].ServerMetrics, _ = metrics.QueryNodeCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime)
-				sr.Results[i].ClientPodCPU, _ = metrics.TopPodCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime)
-				sr.Results[i].ServerPodCPU, _ = metrics.TopPodCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime)
+				if len(npr.ClientNodeInfo.Hostname) > 0 && len(npr.ServerNodeInfo.Hostname) > 0 {
+					sr.Results[i].ClientMetrics, _ = metrics.QueryNodeCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime)
+					sr.Results[i].ServerMetrics, _ = metrics.QueryNodeCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime)
+					sr.Results[i].ClientPodCPU, _ = metrics.TopPodCPU(npr.ClientNodeInfo, pcon, npr.StartTime, npr.EndTime)
+					sr.Results[i].ServerPodCPU, _ = metrics.TopPodCPU(npr.ServerNodeInfo, pcon, npr.StartTime, npr.EndTime)
+				}
 			}
 		}
 
@@ -230,13 +285,22 @@ var rootCmd = &cobra.Command{
 		}
 
 		if len(searchURL) > 1 {
+			var esClient *indexers.Indexer
 			jdocs, err := archive.BuildDocs(sr, uid)
 			if err != nil {
 				log.Fatal(err)
 			}
-			esClient, err := archive.Connect(searchURL, index, true)
-			if err != nil {
-				log.Fatal(err)
+			if len(searchIndex) > 1 {
+				esClient, err = archive.Connect(searchURL, searchIndex, true)
+				if err != nil {
+					log.Fatal(err)
+				}
+			} else {
+				esClient, err = archive.Connect(searchURL, index, true)
+				if err != nil {
+					log.Fatal(err)
+				}
+
 			}
 			log.Infof("Indexing [%d] documents in %s with UUID %s", len(jdocs), index, uid)
 			resp, err := (*esClient).Index(jdocs, indexers.IndexingOpts{})
@@ -305,14 +369,17 @@ func cleanup(client *kubernetes.Clientset) {
 }
 
 // executeWorkload executes the workload and returns the result data.
-func executeWorkload(nc config.Config, s config.PerfScenarios, hostNet bool, driverName string) result.Data {
+func executeWorkload(nc config.Config,
+	s config.PerfScenarios,
+	hostNet bool,
+	driverName string, virt bool) result.Data {
 	serverIP := ""
 	Client := s.Client
 	var driver drivers.Driver
 	if nc.Service {
-		if iperf3 {
+		if driverName == "iperf3" {
 			serverIP = s.IperfService.Spec.ClusterIP
-		} else if uperf {
+		} else if driverName == "uperf" {
 			serverIP = s.UperfService.Spec.ClusterIP
 		} else {
 			serverIP = s.NetperfService.Spec.ClusterIP
@@ -360,7 +427,7 @@ func executeWorkload(nc config.Config, s config.PerfScenarios, hostNet bool, dri
 			log.Warnf("Test %s is not supported with driver %s. Skipping.", nc.Profile, npr.Driver)
 			return npr
 		}
-		r, err := driver.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP)
+		r, err := driver.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP, &s)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -372,7 +439,7 @@ func executeWorkload(nc config.Config, s config.PerfScenarios, hostNet bool, dri
 			// Retry the current test.
 			for try < retry {
 				log.Warn("Rerunning test.")
-				r, err := driver.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP)
+				r, err := driver.Run(s.ClientSet, s.RestConfig, nc, Client, serverIP, &s)
 				if err != nil {
 					log.Error(err)
 					continue
@@ -412,12 +479,14 @@ func main() {
 	rootCmd.Flags().BoolVar(&clean, "clean", true, "Clean-up resources created by k8s-netperf")
 	rootCmd.Flags().BoolVar(&json, "json", false, "Instead of human-readable output, return JSON to stdout")
 	rootCmd.Flags().BoolVar(&nl, "local", false, "Run network performance tests with Server-Pods/Client-Pods on the same Node")
+	rootCmd.Flags().BoolVar(&vm, "vm", false, "Launch Virtual Machines instead of pods for client/servers")
 	rootCmd.Flags().BoolVar(&acrossAZ, "across", false, "Place the client and server across availability zones")
 	rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)")
 	rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log")
 	rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL")
 	rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID")
 	rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port")
+	rootCmd.Flags().StringVar(&searchIndex, "index", "", "OpenSearch Index to save the results to, defaults to k8s-netperf")
 	rootCmd.Flags().BoolVar(&showMetrics, "metrics", false, "Show all system metrics retrieved from prom")
 	rootCmd.Flags().Float64Var(&tcpt, "tcp-tolerance", 10, "Allowed %diff from hostNetwork to podNetwork, anything above tolerance will result in k8s-netperf exiting 1.")
 	rootCmd.Flags().BoolVar(&version, "version", false, "k8s-netperf version")

go.mod

@@ -1,28 +1,32 @@
 module github.com/cloud-bulldozer/k8s-netperf
 
-go 1.18
+go 1.19
 
 require (
 	github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794
 	github.com/cloud-bulldozer/go-commons v1.0.16
 	github.com/google/uuid v1.3.0
+	github.com/melbahja/goph v1.4.0
 	github.com/montanaflynn/stats v0.6.6
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/prometheus/common v0.44.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.6.1
+	golang.org/x/crypto v0.14.0
 	golang.org/x/text v0.13.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.4
 	k8s.io/apimachinery v0.28.4
 	k8s.io/client-go v0.28.4
-	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
+	k8s.io/utils v0.0.0-20230505201702-9f6742963106
+	kubevirt.io/api v1.2.2
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
@@ -36,13 +40,18 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kr/fs v0.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-runewidth v0.0.9 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opensearch-project/opensearch-go v1.1.0 // indirect
+	github.com/openshift/api v0.0.0-20230503133300-8bbcb7ca7183 // indirect
+	github.com/openshift/custom-resource-status v1.1.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pkg/sftp v1.13.5 // indirect
 	github.com/prometheus/client_golang v1.15.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/net v0.17.0 // indirect
@@ -54,8 +63,11 @@ require (
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	k8s.io/apiextensions-apiserver v0.26.3 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	kubevirt.io/containerized-data-importer-api v1.57.0-alpha1 // indirect
+	kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect