run-test-cases.yml

name: Test K3s, Kubernetes, and MicroK8s

on:
  workflow_dispatch:
    inputs:
  pull_request:
    branches: [main]
    paths:
      - test/run-end-to-end.py
      - test/run-conservation-of-broker-pod.py
      - test/run-helm-install-delete.py
      - test/run-webhook.py
      - test/shared_test_code.py
      - .github/workflows/run-test-cases.yml
      - build/containers/Dockerfile.agent
      - build/containers/Dockerfile.controller
      - deployment/helm/**
      - agent/**
      - controller/**
      - shared/**
      - version.txt
      - build/akri-containers.mk
      - Makefile
  push:
    branches: [main]
    paths:
      - test/run-end-to-end.py
      - test/run-conservation-of-broker-pod.py
      - test/run-helm-install-delete.py
      - test/run-webhook.py
      - test/shared_test_code.py
      - .github/workflows/run-test-cases.yml
      - build/containers/Dockerfile.agent
      - build/containers/Dockerfile.controller
      - deployment/helm/**
      - agent/**
      - controller/**
      - shared/**
      - version.txt
      - build/akri-containers.mk
      - Makefile
  release:
    types:
      - published

jobs:
  build-containers:
    runs-on: ubuntu-18.04
    timeout-minutes: 60

    steps:
      - name: Checkout the head commit of the branch
        uses: actions/checkout@v2
        with:
          persist-credentials: false

      - name: Build local containers for PR tests
        if: startsWith(github.event_name, 'pull_request')
        env:
          BUILD_AMD64: 1
          BUILD_ARM32: 0
          BUILD_ARM64: 0
          BUILD_SLIM_AGENT: 0
          AGENT_FEATURES: "agent-full"
          PACKAGES_TO_EXCLUDE: "akri-udev akri-onvif akri-opcua udev-video-broker debug-echo-discovery-handler onvif-discovery-handler opcua-discovery-handler udev-discovery-handler"
          PREFIX: ghcr.io/project-akri/akri
          LABEL_PREFIX: pr
          CARGO_INCREMENTAL: 0
        run: |
          make akri-build
          make controller-build-amd64
          make agent-full-build-amd64
          make webhook-configuration-build-amd64
          docker save ${PREFIX}/agent-full:${LABEL_PREFIX}-amd64 > agent.tar
          docker save ${PREFIX}/controller:${LABEL_PREFIX}-amd64 > controller.tar
          docker save ${PREFIX}/webhook-configuration:${LABEL_PREFIX}-amd64 > webhook-configuration.tar

      - name: Upload Agent container as artifact
        if: startsWith(github.event_name, 'pull_request')
        uses: actions/upload-artifact@v2
        with:
          name: agent.tar
          path: agent.tar
      - name: Upload Controller container as artifact
        if: startsWith(github.event_name, 'pull_request')
        uses: actions/upload-artifact@v2
        with:
          name: controller.tar
          path: controller.tar
      - name: Upload Webhook-Configuration container as artifact
        if: startsWith(github.event_name, 'pull_request')
        uses: actions/upload-artifact@v2
        with:
          name: webhook-configuration.tar
          path: webhook-configuration.tar

  test-cases:
    needs: build-containers
    runs-on: ubuntu-18.04
    timeout-minutes: 90

    strategy:
      fail-fast: false
      matrix:
        kube:
          - runtime: MicroK8s-1.21
            version: 1.21/stable
          - runtime: MicroK8s-1.22
            version: 1.22/stable
          - runtime: MicroK8s-1.23
            version: 1.23/stable
          - runtime: MicroK8s-1.24
            version: 1.24/stable
          - runtime: K3s-1.21
            version: v1.21.5+k3s1
          - runtime: K3s-1.22
            version: v1.22.6+k3s1
          - runtime: K3s-1.23
            version: v1.23.13+k3s1
          - runtime: K3s-1.24
            version: v1.24.6+k3s1
          - runtime: K3s-1.25
            version: v1.25.2+k3s1
          - runtime: Kubernetes-1.21
            version: 1.21.14-00
          - runtime: Kubernetes-1.22
            version: 1.22.14-00
          - runtime: Kubernetes-1.23
            version: 1.23.11-00
          - runtime: Kubernetes-1.24
            version: 1.24.5-00
          - runtime: Kubernetes-1.25
            version: 1.25.1-00
        test:
          - case: end-to-end
            file: test/run-end-to-end.py
          - case: webhook
            file: test/run-webhook.py

    steps:
      - name: Checkout the head commit of the branch
        uses: actions/checkout@v2
        with:
          persist-credentials: false

      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Install Python kubernetes dependency
        run: |
          python -m pip install --upgrade pip
          pip install kubernetes

      - name: Download Agent container artifact
        if: startsWith(github.event_name, 'pull_request')
        uses: actions/download-artifact@v2
        with:
          name: agent.tar
      - name: Download Controller container artifact
        if: startsWith(github.event_name, 'pull_request')
        uses: actions/download-artifact@v2
        with:
          name: controller.tar
      - name: Download Webhook-Configuration container artifact
        if: startsWith(github.event_name, 'pull_request')
        uses: actions/download-artifact@v2
        with:
          name: webhook-configuration.tar

      - if: startsWith(matrix.kube.runtime, 'K3s')
        name: Install K3s
        env:
          INSTALL_K3S_VERSION: ${{ matrix.kube.version }}
        run: |
          sudo curl -sfL https://get.k3s.io -o install.sh
          sudo chmod +x install.sh
          ./install.sh server --kubelet-arg=eviction-hard="imagefs.available<1%,nodefs.available<1%" --kubelet-arg=eviction-minimum-reclaim="imagefs.available=1%,nodefs.available=1%"
          sudo addgroup k3s-admin
          sudo adduser $USER k3s-admin
          sudo usermod -a -G k3s-admin $USER
          sudo chgrp k3s-admin /etc/rancher/k3s/k3s.yaml
          sudo chmod g+r /etc/rancher/k3s/k3s.yaml
          sudo chmod 666 /etc/rancher/k3s/*
          mkdir -p ~/.kube/ && cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
          echo "--set kubernetesDistro=k3s" > /tmp/k8s_distro_to_test.txt
          echo 'kubectl' > /tmp/runtime_cmd_to_test.txt
          echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt
          until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for k3s to become ready"; sleep 10; done
          if [ "${{ matrix.kube.runtime }}" == "K3s-1.16" ]; then
            mkdir -p /var/lib/kubelet
            if [ -d /var/lib/kubelet/device-plugins ]; then
              sudo rm -rf /var/lib/kubelet/device-plugins
            fi
            sudo ln -s /var/lib/rancher/k3s/agent/kubelet/device-plugins /var/lib/kubelet/device-plugins
          fi

      - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'K3s'))
        name: Import local agent and controller to K3s
        run: |
          sudo k3s ctr image import agent.tar
          sudo k3s ctr image import controller.tar
          sudo k3s ctr image import webhook-configuration.tar

      - if: startsWith(matrix.kube.runtime, 'Kubernetes')
        name: Install Kubernetes
        run: |
          sudo apt-get update -y
          sudo apt-get install -y apt-transport-https ca-certificates curl
          sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
          echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
          sudo apt-get update
          sudo apt-get install -o Dpkg::Options::="--force-overwrite" -y --allow-downgrades kubelet=${{ matrix.kube.version }} kubeadm=${{ matrix.kube.version }} kubectl=${{ matrix.kube.version }} 
          kubectl version && echo "kubectl return code: $?" || echo "kubectl return code: $?"
          kubeadm version && echo "kubeadm return code: $?" || echo "kubeadm return code: $?"
          kubelet --version && echo "kubelet return code: $?" || echo "kubelet return code: $?"
          cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
          overlay
          br_netfilter
          EOF
          sudo modprobe overlay
          sudo modprobe br_netfilter
          cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
          net.bridge.bridge-nf-call-iptables = 1
          net.bridge.bridge-nf-call-ip6tables = 1
          net.ipv4.ip_forward = 1
          EOF
          sudo sysctl --system
          sudo mkdir -p /etc/containerd
          containerd config default | sudo tee /etc/containerd/config.toml
          sudo systemctl restart containerd
          sudo swapoff -a              
          sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --cri-socket=/run/containerd/containerd.sock
          sudo mkdir -p $HOME/.kube
          sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
          sudo chown $(id -u):$(id -g) $HOME/.kube/config
          kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

          # Kubernetes 1.25+ replaces the master label with control plane
          if [ "${{ matrix.kube.version }}" == "1.25.1-00" ]; then
            kubectl taint nodes --all node-role.kubernetes.io/control-plane-
          else
            kubectl taint nodes --all node-role.kubernetes.io/master-
          fi

          # Kubernetes 1.24 has both labels applicable, to be safe tainting both labels
          if [ "${{ matrix.kube.version }}" == "1.24.5-00" ]; then
            kubectl taint nodes --all node-role.kubernetes.io/control-plane-
          fi

          echo '--set kubernetesDistro=k8s' > /tmp/k8s_distro_to_test.txt
          echo 'kubectl' > /tmp/runtime_cmd_to_test.txt
          echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt
          until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for kubernetes to become ready"; sleep 10; done

      - if: startsWith(matrix.kube.runtime, 'Kubernetes')
        name: Output Kubelet, Containerd, Docker Logs
        run: |
          echo "Kubelet Logs:"
          sudo journalctl -xeu kubelet --no-pager
          echo "\nContainerd Logs"
          sudo journalctl -xeu containerd --no-pager
          echo "\nDocker Logs"
          sudo journalctl -xeu docker --no-pager

      - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'Kubernetes'))
        name: Import local agent and controller to Kubernetes
        run: |
          sudo docker load --input agent.tar
          sudo docker load --input controller.tar
          sudo docker load --input webhook-configuration.tar
          sudo docker image ls
          # No longer are using docker for container runtime, need to load to containerd with ctr
          # -n allows kubernetes to see the image
          sudo ctr -n=k8s.io image import agent.tar
          sudo ctr -n=k8s.io image import controller.tar
          sudo ctr -n=k8s.io image import webhook-configuration.tar
          sudo crictl images

      - if: startsWith(matrix.kube.runtime, 'MicroK8s')
        name: Install MicroK8s
        run: |
          set -x
          sudo snap install microk8s --classic --channel=${{ matrix.kube.version }}
          sudo microk8s status --wait-ready
          sudo usermod -a -G microk8s $USER
          sudo ls -la $HOME/.kube
          echo sudo chown $(id -u):$(id -g) $HOME/.kube
          sudo chown -f -R $USER $HOME/.kube --verbose
          sudo sh -c "microk8s.kubectl config view --raw >~/.kube/config"
          sudo cat ~/.kube/config
          sudo microk8s.enable rbac dns
          sudo sed -i 's/memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi/memory.available<25Mi,nodefs.available<50Mi,imagefs.available<50Mi/' /var/snap/microk8s/current/args/kubelet
          sudo systemctl restart snap.microk8s.daemon-kubelet
          until sudo microk8s.status --wait-ready; do sleep 5s; echo "Try again"; done
          echo '--set kubernetesDistro=microk8s' > /tmp/k8s_distro_to_test.txt
          echo 'microk8s kubectl' > /tmp/runtime_cmd_to_test.txt
          echo '~/.kube/config' > /tmp/kubeconfig_path_to_test.txt

      - if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'MicroK8s'))
        name: Import local agent and controller to MicroK8s
        run: |
          sudo microk8s.status --wait-ready
          until sudo microk8s ctr images ls; do sleep 5s; echo "Try again"; done
          sudo microk8s ctr images ls
          sudo microk8s ctr --debug --timeout 60s images import agent.tar
          sudo microk8s ctr --debug --timeout 60s images import controller.tar
          sudo microk8s ctr --debug --timeout 60s images import webhook-configuration.tar
          sudo microk8s ctr images ls

      - name: Add Akri Helm Chart
        run: helm repo add akri-helm-charts https://project-akri.github.io/akri/

      # For push and release, we need to wait for the Helm chart and
      # associated containers to build.
      - if: github.event_name == 'push' || github.event_name == 'release'
        name: Set sleep duration before running script to 2700
        run: echo 2700 > /tmp/sleep_duration.txt

      # For pull_request, use the locally built containers.
      - if: startsWith(github.event_name, 'pull_request')
        name: Tell Helm to use the 'local' labels for container images
        run: |
          git fetch origin main
          git show origin/main:version.txt > /tmp/version_to_test.txt
          echo '--set agent.image.pullPolicy=Never,agent.image.tag=pr-amd64,controller.image.pullPolicy=Never,controller.image.tag=pr-amd64,webhookConfiguration.image.pullPolicy=Never,webhookConfiguration.image.tag=pr-amd64' > /tmp/extra_helm_args.txt
      # For non-PR (i.e. push, release, manual), version.txt is corresponds
      # to an existing Helm chart.
      - if: (!(startsWith(github.event_name, 'pull_request')))
        name: Use current version for push
        run: cat version.txt > /tmp/version_to_test.txt

      # For workflow_dispatch and pull_request, use the files in deployment/helm
      # as basis for helm install ... this enables us to test any changes made to
      # the helm chart files in a PR (where no helm chart is published)
      - if: github.event_name != 'push' && github.event_name != 'release'
        name: Tell Helm to use the files in deployment/helm to build chart
        run: |
          echo './deployment/helm' > /tmp/helm_chart_location.txt
      # For push, use a specific version of the `akri-dev` charts that are built and
      # published by the helm workflow.
      - if: github.event_name == 'push'
        name: Tell Helm to use the `akri-dev` published charts
        run: |
          echo "akri-helm-charts/akri-dev --version $(cat /tmp/version_to_test.txt)" > /tmp/helm_chart_location.txt
      # For release, use a specific version of the `akri` charts that are built and
      # published by the helm workflow.
      - if: github.event_name == 'release'
        name: Tell Helm to use the `akri` published charts
        run: |
          echo "akri-helm-charts/akri --version $(cat /tmp/version_to_test.txt)" > /tmp/helm_chart_location.txt

      - name: Execute test script ${{ matrix.test.file }}
        run: python ${{ matrix.test.file }}

      - name: Upload Agent log as artifact
        if: always()
        uses: actions/upload-artifact@v2
        with:
          name: ${{ matrix.kube.runtime }}-${{ matrix.test.case }}-agent-log
          path: /tmp/agent_log.txt
      - name: Upload controller log as artifact
        if: always()
        uses: actions/upload-artifact@v2
        with:
          name: ${{ matrix.kube.runtime }}-${{ matrix.test.case }}-controller-log
          path: /tmp/controller_log.txt
      - name: Upload webhook log as artifact
        if: always()
        uses: actions/upload-artifact@v2
        with:
          name: ${{ matrix.kube.runtime }}-${{ matrix.test.case }}-webhook-log
          path: /tmp/webhook_log.txt