Merge pull request #642 from openstack-k8s-operators/zuul_inventory

Properly pass the real inventory to all nested plays

ci/playbooks/collect-logs.yml

@@ -46,18 +46,6 @@
                   cp -ra {{ ansible_user_dir }}/ci-framework-data/artifacts . ;
                   cp -ra {{ ansible_user_dir }}/ci-framework-data/tests . || true ;
 
-            - name: Check for Zuul inventory file
-              ansible.builtin.stat:
-                path: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/scenarios/centos-9/zuul_inventory.yml"
-              register: cifmw_zuul_state
-
-            - name: Copy Zuul inventory file
-              when: cifmw_zuul_state.stat.exists | bool
-              ansible.builtin.copy:
-                remote_src: true
-                dest: "{{ ansible_user_dir }}/ci-framework-data/logs"
-                src: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework/scenarios/centos-9/zuul_inventory.yml"
-
             - name: Get SELinux listing
               ansible.builtin.shell:
                 chdir: "{{ ansible_user_dir }}/zuul-output/logs/"

ci/playbooks/content_provider/run.yml

@@ -20,7 +20,7 @@
           -c local ci/playbooks/content_provider/content_provider.yml
           -e @scenarios/centos-9/base.yml
           -e @scenarios/centos-9/content_provider.yml
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
           -e "cifmw_rp_registry_ip={{ cifmw_rp_registry_ip }}"
 
     - name: Include inner ansible vars file

ci/playbooks/dev-workflow.yml

@@ -28,4 +28,4 @@
             -e cifmw_basedir=/opt/cifmw
             -e cifmw_local_env_vm_cpu=1
             -e cifmw_local_env_vm_memory=1024
-            -e @scenarios/centos-9/zuul_inventory.yml
+            -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"

ci/playbooks/dump_zuul_data.yml

@@ -0,0 +1,40 @@
+---
+- name: "Run ci/playbooks/dump_zuul_data.yml"
+  hosts: controller
+  gather_facts: true
+  tasks:
+    - name: Create zuul-output directory
+      ansible.builtin.file:
+        path: "{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters"
+        state: directory
+
+    - name: Slurp Zuul inventory test
+      ansible.builtin.slurp:
+        path: "{{ zuul.executor.log_root }}/zuul-info/inventory.yaml"
+      register: _inventory_yaml
+      delegate_to: localhost
+
+    - name: Save zuul inventory
+      vars:
+        _original_inventory: "{{ _inventory_yaml['content'] | b64decode | from_yaml }}"
+        _localhost_inventory:
+          all:
+            hosts:
+              localhost:
+                ansible_connection: local
+      ansible.builtin.copy:
+        dest: "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
+        content: >-
+          {{
+              _original_inventory |
+              combine(_localhost_inventory, recursive=True) |
+              to_nice_yaml
+          }}
+
+    - name: Save zuul vars
+      vars:
+        _original_inventory: "{{ _inventory_yaml['content'] | b64decode | from_yaml }}"
+      ansible.builtin.copy:
+        content: '{{ _original_inventory.all.vars | to_nice_yaml }}'
+        dest: "{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
+        mode: "0644"

ci/playbooks/e2e-run.yml

@@ -10,14 +10,15 @@
         chdir: "{{ ci_framework_src_dir }}"
         cmd: >-
           ansible-playbook deploy-edpm.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
           -e @scenarios/centos-9/base.yml
           -e @scenarios/centos-9/install_yamls.yml
           {%- if cifmw_extras is defined %}
           {%-   for extra_vars in cifmw_extras %}
           -e "{{   extra_vars }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
           --tags packages
 
     - name: Clean ansible cache
@@ -30,14 +31,15 @@
         chdir: "{{ ci_framework_src_dir }}"
         cmd: >-
           ansible-playbook deploy-edpm.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
           -e @scenarios/centos-9/base.yml
           -e @scenarios/centos-9/install_yamls.yml
           {%- if cifmw_extras is defined %}
           {%-   for extra_vars in cifmw_extras %}
           -e "{{   extra_vars }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
           --tags bootstrap
           --skip-tags packages
 
@@ -51,12 +53,13 @@
         chdir: "{{ ci_framework_src_dir }}"
         cmd: >-
           ansible-playbook deploy-edpm.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
           -e @scenarios/centos-9/base.yml
           -e @scenarios/centos-9/install_yamls.yml
           {%- if cifmw_extras is defined %}
           {%-   for extra_vars in cifmw_extras %}
           -e "{{   extra_vars }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
           --skip-tags bootstrap,packages

ci/playbooks/edpm/run.yml

@@ -20,6 +20,7 @@
         chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
         cmd: >-
           ansible-playbook deploy-edpm.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
           -e @scenarios/centos-9/base.yml
           -e @scenarios/centos-9/edpm_ci.yml
           {%- if edpm_file.stat.exists %}
@@ -30,4 +31,4 @@
           -e "{{   extra_var }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"

ci/playbooks/edpm_baremetal_deployment/run.yml

@@ -20,6 +20,7 @@
         chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
         cmd: >-
           ansible-playbook deploy-edpm.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
           -e @scenarios/centos-9/base.yml
           -e @scenarios/centos-9/edpm_baremetal_deployment_ci.yml
           {%- if edpm_file.stat.exists %}
@@ -30,4 +31,4 @@
           -e "{{   extra_var }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"

ci/playbooks/edpm_build_images/edpm_build_images_content_provider_run.yaml

@@ -16,15 +16,16 @@
       ansible.builtin.command:
         chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
         cmd: >-
-          ansible-playbook ci/playbooks/edpm_build_images/edpm_build_images_content_provider.yaml
+          ansible-playbook -i localhost, -c local
+          ci/playbooks/edpm_build_images/edpm_build_images_content_provider.yaml
           -e @scenarios/centos-9/base.yml
           {%- if cifmw_extras is defined %}
           {%-   for extra_vars in cifmw_extras %}
           -e "{{   extra_vars }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
           -e @scenarios/centos-9/edpm_build_images_content_provider.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
           -e "cifmw_rp_registry_ip={{ cifmw_rp_registry_ip }}"
 
     - name: Include inner ansible vars file

ci/playbooks/edpm_build_images/edpm_image_builder.yml

@@ -12,7 +12,7 @@
     }}
 
 - name: "Run ci/playbooks/edpm_build_images/edpm_image_builder.yml"
-  hosts: "{{ cifmw_zuul_target_host | default('all') }}"
+  hosts: "{{ cifmw_zuul_target_host | default('localhost') }}"
   tasks:
     - name: Filter out host if needed
       when:

ci/playbooks/edpm_build_images/run.yml

@@ -16,12 +16,13 @@
       ansible.builtin.command:
         chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
         cmd: >-
-          ansible-playbook ci/playbooks/edpm_build_images/edpm_image_builder.yml
+          ansible-playbook -i localhost, -c local
+          ci/playbooks/edpm_build_images/edpm_image_builder.yml
           -e @scenarios/centos-9/base.yml
           {%- if cifmw_extras is defined %}
           {%-   for extra_vars in cifmw_extras %}
           -e "{{   extra_vars }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
           --tags packages,edpm_build_img

ci/playbooks/kuttl/run.yml

@@ -7,6 +7,7 @@
         chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
         cmd: >-
           ansible-playbook ci/playbooks/kuttl/e2e-kuttl.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
           -e @scenarios/centos-9/base.yml
           -e @scenarios/centos-9/ci.yml
           -e @scenarios/centos-9/kuttl.yml
@@ -15,4 +16,4 @@
           -e "{{   extra_vars }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"

ci/playbooks/multinode-customizations.yml

@@ -94,16 +94,6 @@
   name: "Tweak Controller"
   gather_facts: true
   tasks:
-    - name: Create zuul-output directory
-      ansible.builtin.file:
-        path: "{{ ansible_user_dir }}/ci-framework-data/artifacts"
-        state: directory
-
-    - name: Save zuul inventory
-      ansible.builtin.copy:
-        dest: "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
-        src: "{{ inventory_file }}"
-
     - name: Install other packages
       become: true
       ansible.builtin.package:

ci/playbooks/tcib/run.yml

@@ -22,6 +22,7 @@
         chdir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/ci-framework"
         cmd: >-
           ansible-playbook ci/playbooks/tcib/tcib.yml
+          -i "{{ ansible_user_dir }}/ci-framework-data/artifacts/zuul_inventory.yml"
           -e @scenarios/centos-9/base.yml
           -e @scenarios/centos-9/tcib.yml
           -e "cifmw_rp_registry_ip={{ node_ip }}"
@@ -30,7 +31,7 @@
           -e "{{   extra_vars }}"
           {%-   endfor %}
           {%- endif %}
-          -e @scenarios/centos-9/zuul_inventory.yml
+          -e "@{{ ansible_user_dir }}/ci-framework-data/artifacts/parameters/zuul-params.yml"
 
     - name: Return Zuul Data
       ansible.builtin.debug:

ci_framework/hooks/playbooks/fetch_compute_facts.yml

@@ -8,7 +8,12 @@
       ansible.builtin.shell:
         cmd: |
           ssh-keyscan {{ hostvars[item].ansible_host }} >> ~/.ssh/known_hosts
-      loop: "{{ hostvars.keys() }}"
+      loop: >-
+        {{
+          hostvars | dict2items |
+          selectattr("value.ansible_host", "defined") |
+          map(attribute="key")
+        }}
       loop_control:
         label: "{{ item }}"
 

ci_framework/playbooks/01-bootstrap.yml

@@ -61,10 +61,3 @@
       ansible.builtin.copy:
         dest: "{{ cifmw_basedir|default(ansible_user_dir ~ '/ci-framework-data') }}/artifacts/parameters/custom-params.yml"
         content: "{{ ci_framework_params | to_nice_yaml }}"
-
-    - name: Save zuul parameter if exists
-      when:
-        - zuul is defined
-      ansible.builtin.copy:
-        dest: "{{ cifmw_basedir|default(ansible_user_dir ~ '/ci-framework-data') }}/artifacts/parameters/zuul-params.yml"
-        content: "{{ {'zuul': zuul} | to_nice_yaml }}"

zuul.d/adoption.yaml

@@ -21,7 +21,7 @@
     pre-run:
       - ci/playbooks/multinode-customizations.yml
       - ci/playbooks/e2e-prepare.yml
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
     post-run:
       - ci/playbooks/multinode-autohold.yml
     vars:

zuul.d/base.yaml

@@ -70,7 +70,7 @@
     pre-run:
       - ci/playbooks/content_provider/pre.yml
       - ci/playbooks/e2e-prepare.yml
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
     run:
       - ci/playbooks/content_provider/run.yml
     post-run: ci/playbooks/collect-logs.yml
@@ -121,7 +121,7 @@
     pre-run:
       - ci/playbooks/multinode-customizations.yml
       - ci/playbooks/e2e-prepare.yml
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
     post-run:
       - ci/playbooks/e2e-collect-logs.yml
       - ci/playbooks/collect-logs.yml
@@ -216,7 +216,7 @@
       - zuul: github.com/openstack-k8s-operators/ci-framework
     pre-run:
       - ci/playbooks/e2e-prepare.yml
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
     post-run:
       - ci/playbooks/e2e-collect-logs.yml
       - ci/playbooks/collect-logs.yml

zuul.d/edpm_build_images.yaml

@@ -10,10 +10,11 @@
     pre-run:
       - ci/playbooks/molecule-prepare.yml
     run:
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
       - ci/playbooks/edpm_build_images/run.yml
     post-run: ci/playbooks/collect-logs.yml
     vars:
+      cifmw_zuul_target_host: controller
       cifmw_repo_setup_branch: antelope
       cifmw_repo_setup_output: "/etc/yum.repos.d/"
       ansible_user_dir: "{{ lookup('env', 'HOME') }}"

zuul.d/edpm_build_images_content_provider.yaml

@@ -12,7 +12,7 @@
       - ci/playbooks/content_provider/pre.yml
     run:
       - ci/playbooks/e2e-prepare.yml
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
       - ci/playbooks/edpm_build_images/edpm_build_images_content_provider_run.yaml
     post-run: ci/playbooks/collect-logs.yml
     vars:

zuul.d/end-to-end.yaml

@@ -10,7 +10,7 @@
       registry_login_enabled: false
     pre-run:
       - ci/playbooks/e2e-prepare.yml
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
     post-run:
       - ci/playbooks/e2e-collect-logs.yml
       - ci/playbooks/collect-logs.yml
@@ -67,6 +67,6 @@
     pre-run:
       - ci/playbooks/molecule-prepare.yml
     run:
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
       - ci/playbooks/dev-workflow.yml
     post-run: ci/playbooks/collect-logs.yml

zuul.d/kuttl.yaml

@@ -11,7 +11,7 @@
     pre-run:
       - ci/playbooks/e2e-prepare.yml
     run:
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
       - ci/playbooks/kuttl/run.yml
     post-run:
       - ci/playbooks/e2e-collect-logs.yml

zuul.d/kuttl_multinode.yaml

@@ -45,7 +45,7 @@
     pre-run:
       - ci/playbooks/e2e-prepare.yml
     run:
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
       - ci/playbooks/kuttl/run.yml
     post-run:
       - ci/playbooks/collect-logs.yml

zuul.d/tcib.yaml

@@ -13,7 +13,7 @@
     pre-run:
       - ci/playbooks/content_provider/pre.yml
       - ci/playbooks/e2e-prepare.yml
-      - ci/playbooks/dump_zuul_vars.yml
+      - ci/playbooks/dump_zuul_data.yml
     run:
       - ci/playbooks/tcib/run.yml
     post-run: ci/playbooks/collect-logs.yml