Update uperf process (#840)

* uperf: wait for vms update * Update base image * Update pod starting clients check
cloud-bulldozer · Jan 8, 2025 · 7b4f818 · 7b4f818
1 parent deb7c8b
commit 7b4f818
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 28 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM quay.io/operator-framework/ansible-operator:v1.32.0
+FROM quay.io/operator-framework/ansible-operator:v1.37.0
 USER root
 
 COPY requirements.yml ${HOME}/requirements.yml
@@ -8,8 +8,8 @@ RUN ansible-galaxy collection list
 RUN ansible-galaxy collection install -r ${HOME}/requirements.yml \
  && chmod -R ug+rwx ${HOME}/.ansible
 
-COPY image_resources/centos8-appstream.repo /etc/yum.repos.d/centos8-appstream.repo
-RUN dnf install -y --nodocs redis openssl --enablerepo=centos8-appstream-* && dnf clean all
+COPY image_resources/centos9-appstream.repo /etc/yum.repos.d/centos9-appstream.repo
+RUN dnf install -y --nodocs redis openssl --enablerepo=centos9-appstream-* --refresh && dnf clean all
 
 COPY resources/kernel-cache-drop-daemonset.yaml /opt/kernel_cache_dropper/
 COPY watches.yaml ${HOME}/watches.yaml

diff --git a/image_resources/centos9-appstream.repo b/image_resources/centos9-appstream.repo
@@ -0,0 +1,17 @@
+[centos9-appstream-x86_64]
+name=CentOS-9-Appstream-x86_64
+baseurl=http://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/
+enabled=0
+gpgcheck=0
+
+[centos9-appstream-aarch64]
+name=CentOS-9-Appstream-aarch64
+baseurl=http://mirror.stream.centos.org/9-stream/AppStream/aarch64/os/
+enabled=0
+gpgcheck=0
+
+[centos9-appstream-ppc64le]
+name=CentOS-9-Appstream-ppc64le
+baseurl=http://mirror.stream.centos.org/9-stream/AppStream/ppc64le/os/
+enabled=0
+gpgcheck=0
diff --git a/roles/uperf/tasks/main.yml b/roles/uperf/tasks/main.yml
@@ -10,12 +10,16 @@
   register: serviceip
   when: workload_args.serviceip is defined and workload_args.serviceip
 
+- name: Get Workload Pairs
+  set_fact:
+    workload_pairs: "{{ workload_args.pair|default('1')|int }}"
+
 - block:
 
   - name: Create service for server pods
     k8s:
       definition: "{{ lookup('template', 'service.yml.j2') | from_yaml }}"
-    with_sequence: start=0 count={{ workload_args.pair | default('1')|int }}
+    with_sequence: start=0 count={{ workload_pairs }}
     when: 
       - workload_args.serviceip is defined and workload_args.serviceip
       - ( workload_args.servicetype | default("clusterip") == "clusterip" ) or 
@@ -24,7 +28,7 @@
   - name: Create metal LB service for server pods
     k8s:
       definition: "{{ lookup('template', 'service_metallb.yml.j2') | from_yaml }}"
-    with_sequence: start=0 count={{ workload_args.pair | default('1')|int }}
+    with_sequence: start=0 count={{ workload_pairs }}
     when: 
       - workload_args.serviceip is defined and workload_args.serviceip
       - workload_args.servicetype | default("clusterip") == "metallb"
@@ -33,7 +37,7 @@
     k8s:
       definition: "{{ lookup('template', 'server.yml.j2') | from_yaml }}"
     register: servers
-    with_sequence: start=0 count={{ workload_args.pair | default('1')|int }}
+    with_sequence: start=0 count={{ workload_pairs }}
 
   - include_role:
       name: benchmark_state
@@ -49,7 +53,7 @@
     k8s:
       definition: "{{ lookup('template', 'server_vm.yml.j2') | from_yaml }}"
     register: servers
-    with_sequence: start=0 count={{ workload_args.pair | default('1')|int }}
+    with_sequence: start=0 count={{ workload_pairs }}
 
   - include_role:
       name: benchmark_state
@@ -70,19 +74,21 @@
         - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}
     register: server_pods
 
+  - name: Get number of running server pods
+    set_fact:
+      running_pod_count: "{{ server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length|int }}"
+
   - include_role:
       name: benchmark_state
       tasks_from: set_state
     vars:
       state: "Starting Clients"
-    when: "workload_args.pair|default('1')|int == server_pods | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length"
-
+    when: workload_pairs == running_pod_count
 
   when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "pod"
 
 - block:
-
-  - name: Wait for vms to be running....
+  - name: Get Server VMs
     k8s_info:
       kind: VirtualMachineInstance
       api_version: kubevirt.io/v1alpha3
@@ -91,19 +97,31 @@
         - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}
     register: server_vms
 
+  - name: Get Running VMs
+    set_fact:
+      running_vm_count: "{{ server_vms | json_query('resources[].status[]') | selectattr('phase','match','Running') | list | length | int }}"
+
+  - name: Get VMs with ip addresses
+    set_fact:
+      server_vms_with_ips: "{{ server_vms | json_query('resources[].status.interfaces[0]') | selectattr('ipAddress', 'defined') | map(attribute='ipAddress') | list | length | int }}"
+
+  - name: Determine if can start clients
+    set_fact:
+      start_clients: "{{ (workload_pairs == running_vm_count) and (workload_pairs == server_vms_with_ips) }}"
+
   - include_role:
       name: benchmark_state
       tasks_from: set_state
     vars:
       state: "Starting Clients"
-    when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int  == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)"
+    when: start_clients
 
   - name: blocking client from running uperf
     command: "redis-cli set start-{{ trunc_uuid }} false"
     with_items: "{{ server_vms.resources }}"
-    when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int  == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)"
+    when: start_clients
 
-  when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1
+  when: benchmark_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm"
 
 - block:
 
@@ -148,11 +166,16 @@
         - workload_args.serviceip|default(False) == True and serviceip.resources|length > 0
         - workload_args.servicetype | default("clusterip") != "nodeport"
 
+    - include_role:
+        name: benchmark_state
+        tasks_from: set_state
+      vars:
+        state: "Waiting for Clients"
+
     when: resource_kind == "pod"
 
   - block:
-
-    - name: Wait for vms to be running....
+    - name: Get Server VMs
       k8s_info:
         kind: VirtualMachineInstance
         api_version: kubevirt.io/v1alpha3
@@ -161,25 +184,39 @@
           - type = {{ ansible_operator_meta.name }}-bench-server-{{ trunc_uuid }}
       register: server_vms
 
+    - name: Get Running VMs
+      set_fact:
+        running_vm_count: "{{ server_vms | json_query('resources[].status[]') | selectattr('phase','match','Running') | list | length | int }}"
+
+    - name: Get VMs with ip addresses
+      set_fact:
+        server_vms_with_ips: "{{ server_vms | json_query('resources[].status.interfaces[0]') | selectattr('ipAddress', 'defined') | map(attribute='ipAddress') | list | length | int }}"
+
+    - name: Determine if can start clients
+      set_fact:
+        start_clients: "{{ (workload_pairs == running_vm_count) and (workload_pairs == server_vms_with_ips) }}"
 
     - name: Generate uperf test files
       k8s:
         definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}"
       with_items: "{{ server_vms.resources }}"
+      when: start_clients
 
     - name: Start Client(s)
       k8s:
         definition: "{{ lookup('template', 'workload_vm.yml.j2') | from_yaml }}"
       with_indexed_items: "{{ server_vms.resources }}"
-      when: server_vms.resources|length > 0
+      when: start_clients
+
+    - include_role:
+        name: benchmark_state
+        tasks_from: set_state
+      vars:
+        state: "Waiting for Clients"
+      when: start_clients
 
     when: resource_kind == "vm"
 
-  - include_role:
-      name: benchmark_state
-      tasks_from: set_state
-    vars:
-      state: "Waiting for Clients"
 
   when: benchmark_state.resources[0].status.state == "Starting Clients"
 

diff --git a/roles/uperf/templates/configmap_script.yml.j2 b/roles/uperf/templates/configmap_script.yml.j2
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: uperf-run-script-{{item.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}
+  name: uperf-run-script-{{item.status.interfaces[0].ipAddress | replace(".", "-") }}-{{ trunc_uuid }}
   namespace: '{{ operator_namespace }}'
 data:
   run_script.sh : |

diff --git a/roles/uperf/templates/workload.yml.j2 b/roles/uperf/templates/workload.yml.j2
@@ -7,12 +7,12 @@ metadata:
 {% if workload_args.servicetype | default("clusterip") == "nodeport" %}
   name: 'uperf-client-{{item.status.hostIP}}-{{ item.metadata.labels.index|int }}-{{ trunc_uuid }}'
 {% elif workload_args.servicetype | default("clusterip") == "metallb" or workload_args.servicetype | default("clusterip") == "loadbalancer" %}
-  name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip}}-{{ trunc_uuid }}'
+  name: 'uperf-client-{{item.status.loadBalancer.ingress[0].ip | replace(".", "-") }}-{{ trunc_uuid }}'
 {% else %}
-  name: 'uperf-client-{{item.spec.clusterIP}}-{{ trunc_uuid }}'
+  name: 'uperf-client-{{item.spec.clusterIP | replace(".", "-")}}-{{ trunc_uuid }}'
 {% endif %}
 {% else %}
-  name: 'uperf-client-{{item.status.podIP}}-{{ trunc_uuid }}'
+  name: 'uperf-client-{{item.status.podIP | replace(".", "-")}}-{{ trunc_uuid }}'
 {% endif %}
   namespace: '{{ operator_namespace }}'
 {% if workload_args.annotations is defined or workload_args.server_annotations is defined %}

diff --git a/roles/uperf/templates/workload_vm.yml.j2 b/roles/uperf/templates/workload_vm.yml.j2
@@ -2,7 +2,7 @@
 apiVersion: kubevirt.io/v1alpha3
 kind: VirtualMachineInstance
 metadata:
-  name: 'uperf-client-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}'
+  name: 'uperf-client-{{item.1.status.interfaces[0].ipAddress | replace(".", "-") }}-{{ trunc_uuid }}'
   namespace: '{{ operator_namespace }}'
   labels:
     benchmark-uuid: {{ uuid }}
@@ -93,6 +93,6 @@ spec:
       name: uperf-test-{{ item.0 }}-{{ trunc_uuid }}
     name: app-config-disk
   - configMap:
-      name: uperf-run-script-{{item.1.status.interfaces[0].ipAddress}}-{{ trunc_uuid }}
+      name: uperf-run-script-{{item.1.status.interfaces[0].ipAddress | replace(".", "-") }}-{{ trunc_uuid }}
     name: run-config-disk
 status: {}