diff --git a/cmd/k8s-netperf/k8s-netperf.go b/cmd/k8s-netperf/k8s-netperf.go index 44fcbebf..15270caa 100644 --- a/cmd/k8s-netperf/k8s-netperf.go +++ b/cmd/k8s-netperf/k8s-netperf.go @@ -43,6 +43,7 @@ var ( full bool vm bool debug bool + bridge string promURL string id string searchURL string @@ -164,6 +165,13 @@ var rootCmd = &cobra.Command{ } s.KClient = kclient s.DClient = dynClient + if len(bridge) > 0 { + err := k8s.DeployNADBridge(s.DClient, bridge) + if err != nil { + log.Error(err) + } + s.Bridge = true + } } // Build the SUT (Deployments) @@ -384,6 +392,9 @@ func executeWorkload(nc config.Config, } else { serverIP = s.NetperfService.Spec.ClusterIP } + //when using a bridge, ip is static + } else if s.Bridge { + serverIP = "10.10.10.14" } else { if hostNet { serverIP = s.ServerHost.Items[0].Status.PodIP @@ -481,6 +492,7 @@ func main() { rootCmd.Flags().BoolVar(&acrossAZ, "across", false, "Place the client and server across availability zones") rootCmd.Flags().BoolVar(&full, "all", false, "Run all tests scenarios - hostNet and podNetwork (if possible)") rootCmd.Flags().BoolVar(&debug, "debug", false, "Enable debug log") + rootCmd.Flags().StringVar(&bridge, "bridge", "", "Name of the NNCP to be used for creating bridge interface - VM only.") rootCmd.Flags().StringVar(&promURL, "prom", "", "Prometheus URL") rootCmd.Flags().StringVar(&id, "uuid", "", "User provided UUID") rootCmd.Flags().StringVar(&searchURL, "search", "", "OpenSearch URL, if you have auth, pass in the format of https://user:pass@url:port") diff --git a/pkg/config/config.go b/pkg/config/config.go index 8195d2c0..e86620e0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,6 +38,7 @@ type PerfScenarios struct { Configs []Config VM bool VMHost string + Bridge bool ServerNodeInfo metrics.NodeInfo ClientNodeInfo metrics.NodeInfo Client apiv1.PodList diff --git a/pkg/k8s/kubernetes.go b/pkg/k8s/kubernetes.go index 8d681f8d..f0d34e99 100644 --- a/pkg/k8s/kubernetes.go +++ b/pkg/k8s/kubernetes.go @@ -12,9 +12,12 @@ import ( corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" ) @@ -124,6 +127,36 @@ func BuildInfra(client *kubernetes.Clientset) error { return nil } +// Create a NetworkAttachcmentDefinition object for a bridge connection +func DeployNADBridge(dyn *dynamic.DynamicClient, bridgeName string) error { + nadBridge := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "k8s.cni.cncf.io/v1", + "kind": "NetworkAttachmentDefinition", + "metadata": map[string]interface{}{ + "name": "br-netperf", + "namespace": "netperf", + "annotations": map[string]interface{}{ + "k8s.v1.cni.cncf.io/resourceName": "bridge.network.kubevirt.io/" + bridgeName, + }, + }, + "spec": map[string]interface{}{ + "config": `{"cniVersion": "0.3.1", "type": "bridge", "name": "br-netperf", "bridge": "` + bridgeName + `"}`, + }, + }, + } + gvr := schema.GroupVersionResource{ + Group: "k8s.cni.cncf.io", + Version: "v1", + Resource: "network-attachment-definitions", + } + _, err := dyn.Resource(gvr).Namespace(namespace).Create(context.TODO(), nadBridge, metav1.CreateOptions{}) + if err != nil { + return err + } + return nil +} + // BuildSUT Build the k8s env to run network performance tests func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error { var netperfDataPorts []int32 @@ -451,7 +484,7 @@ func BuildSUT(client *kubernetes.Clientset, s *config.PerfScenarios) error { // launchServerVM will create the ServerVM with the specific node and pod affinity. func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff) + _, err := CreateVMServer(perf.KClient, serverRole, serverRole, *podAff, *nodeAff, perf.Bridge) if err != nil { return err } @@ -476,7 +509,7 @@ func launchServerVM(perf *config.PerfScenarios, name string, podAff *corev1.PodA // launchClientVM will create the ClientVM with the specific node and pod affinity. func launchClientVM(perf *config.PerfScenarios, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) error { - host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff) + host, err := CreateVMClient(perf.KClient, perf.ClientSet, perf.DClient, name, podAff, nodeAff, perf.Bridge) if err != nil { return err } diff --git a/pkg/k8s/kubevirt.go b/pkg/k8s/kubevirt.go index 6834ee4c..f96addf1 100644 --- a/pkg/k8s/kubevirt.go +++ b/pkg/k8s/kubevirt.go @@ -158,7 +158,7 @@ func exposeService(client *kubernetes.Clientset, dynamicClient *dynamic.DynamicC // CreateVMClient takes in the affinity rules and deploys the VMI func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Clientset, - dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity) (string, error) { + dyn *dynamic.DynamicClient, name string, podAff *corev1.PodAntiAffinity, nodeAff *corev1.NodeAffinity, bridge bool) (string, error) { label := map[string]string{ "app": name, "role": name, @@ -171,6 +171,7 @@ func CreateVMClient(kclient *kubevirtv1.KubevirtV1Client, client *kubernetes.Cli if err != nil { return "", err } + netData := "{}" data := fmt.Sprintf(`#cloud-config users: - name: fedora @@ -184,7 +185,7 @@ chpasswd: { expire: False } runcmd: - export HOME=/home/fedora - dnf install -y --nodocs uperf iperf3 git ethtool automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - - git clone https://github.com/HewlettPackard/netperf + - git clone https://github.com/HewlettPackard/netperf.git - cd netperf - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4 - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff @@ -196,7 +197,43 @@ runcmd: - curl -o /usr/bin/super-netperf https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/super-netperf - chmod 0777 /usr/bin/super-netperf `, ssh) - _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff) + interfaces := []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }, + } + networks := []v1.Network{ + { + Name: "default", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + if bridge { + interfaces = append(interfaces, v1.Interface{ + Name: "br-netperf", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }) + networks = append(networks, v1.Network{ + Name: "br-netperf", + NetworkSource: v1.NetworkSource{ + Multus: &v1.MultusNetwork{ + NetworkName: "netperf/br-netperf", + }, + }, + }) + netData = `version: 2 +ethernets: + eth1: + addresses: [ 10.10.10.12/24 ]` + } + _, err = CreateVMI(kclient, name, label, b64.StdEncoding.EncodeToString([]byte(data)), *podAff, *nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) if err != nil { return "", err } @@ -213,11 +250,12 @@ runcmd: // CreateVMServer will take the pod and node affinity and deploy the VMI func CreateVMServer(client *kubevirtv1.KubevirtV1Client, name string, role string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, bridge bool) (*v1.VirtualMachineInstance, error) { label := map[string]string{ "app": name, "role": role, } + netData := "{}" dirname, err := os.UserHomeDir() if err != nil { return nil, err @@ -239,7 +277,7 @@ chpasswd: { expire: False } runcmd: - dnf install -y --nodocs uperf iperf3 git ethtool - dnf install -y --nodocs automake gcc bc lksctp-tools-devel texinfo --enablerepo=* - - git clone https://github.com/HewlettPackard/netperf + - git clone https://github.com/HewlettPackard/netperf.git - cd netperf - git reset --hard 3bc455b23f901dae377ca0a558e1e32aa56b31c4 - curl -o netperf.diff https://raw.githubusercontent.com/cloud-bulldozer/k8s-netperf/main/containers/netperf.diff @@ -252,12 +290,48 @@ runcmd: - iperf3 -s -p %d & - netserver & `, string(ssh), UperfServerCtlPort, IperfServerCtlPort) - return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff) + interfaces := []v1.Interface{ + { + Name: "default", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }, + } + networks := []v1.Network{ + { + Name: "default", + NetworkSource: v1.NetworkSource{ + Pod: &v1.PodNetwork{}, + }, + }, + } + if bridge { + interfaces = append(interfaces, v1.Interface{ + Name: "br-netperf", + InterfaceBindingMethod: v1.InterfaceBindingMethod{ + Bridge: &v1.InterfaceBridge{}, + }, + }) + networks = append(networks, v1.Network{ + Name: "br-netperf", + NetworkSource: v1.NetworkSource{ + Multus: &v1.MultusNetwork{ + NetworkName: "netperf/br-netperf", + }, + }, + }) + netData = `version: 2 +ethernets: + eth1: + addresses: [ 10.10.10.14/24 ]` + } + return CreateVMI(client, name, label, b64.StdEncoding.EncodeToString([]byte(data)), podAff, nodeAff, interfaces, networks, b64.StdEncoding.EncodeToString([]byte(netData))) } // CreateVMI creates the desired Virtual Machine instance with the cloud-init config with affinity. func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[string]string, b64data string, podAff corev1.PodAntiAffinity, - nodeAff corev1.NodeAffinity) (*v1.VirtualMachineInstance, error) { + nodeAff corev1.NodeAffinity, interfaces []v1.Interface, networks []v1.Network, netDatab64 string) (*v1.VirtualMachineInstance, error) { delSeconds := int64(0) mutliQ := true vmi, err := client.VirtualMachineInstances(namespace).Create(context.TODO(), &v1.VirtualMachineInstance{ @@ -300,8 +374,10 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin }, }, }, + Interfaces: interfaces, }, }, + Networks: networks, Volumes: []v1.Volume{ v1.Volume{ Name: "disk0", @@ -315,7 +391,8 @@ func CreateVMI(client *kubevirtv1.KubevirtV1Client, name string, label map[strin Name: "cloudinit", VolumeSource: v1.VolumeSource{ CloudInitNoCloud: &v1.CloudInitNoCloudSource{ - UserDataBase64: b64data, + UserDataBase64: b64data, + NetworkDataBase64: netDatab64, }, }, },