From 6f1554c2891de424a7d7a924ce4ec23c02e9b2e6 Mon Sep 17 00:00:00 2001
From: Steve Brasier <33413598+sjpb@users.noreply.github.com>
Date: Thu, 24 Oct 2024 10:20:57 +0100
Subject: [PATCH] Add workflow to cleanup CI clusters (#451)

* add CI to cleanup ci clusters

* change CI cluster cleanup to be earlier to save $

* don't trigger deployment/reimage workflow on changes to workflows other than itself

* Apply fixes to CI cleanup workflow from code review

Co-authored-by: bertiethorpe <84867280+bertiethorpe@users.noreply.github.com>

---------

Co-authored-by: bertiethorpe <84867280+bertiethorpe@users.noreply.github.com>
---
 .github/workflows/nightly-cleanup.yml | 71 +++++++++++++++++++++++++++
 .github/workflows/stackhpc.yml        |  4 ++
 2 files changed, 75 insertions(+)
 create mode 100644 .github/workflows/nightly-cleanup.yml

diff --git a/.github/workflows/nightly-cleanup.yml b/.github/workflows/nightly-cleanup.yml
new file mode 100644
index 000000000..9bfc5230a
--- /dev/null
+++ b/.github/workflows/nightly-cleanup.yml
@@ -0,0 +1,71 @@
+name: Cleanup CI clusters
+on:
+  workflow_dispatch:
+      inputs:
+        ci_cloud:
+          description: 'Select the CI_CLOUD'
+          required: true
+          type: choice
+          options:
+            - LEAFCLOUD
+            - SMS
+            - ARCUS
+  schedule:
+    - cron: '0 20 * * *'  # Run at 8PM - image sync runs at midnight
+
+jobs:
+  ci_cleanup:
+    name: ci-cleanup
+    concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.cloud }}
+    strategy:
+      fail-fast: false
+      matrix:
+        cloud:
+          - LEAFCLOUD
+          - SMS
+          - ARCUS
+    runs-on: ubuntu-22.04
+    env:
+      OS_CLOUD: openstack
+      CI_CLOUD: ${{ matrix.cloud }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Record which cloud CI is running on
+        run: |
+          echo CI_CLOUD: ${{ env.CI_CLOUD }}
+
+      - name: Setup environment
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          pip install -U pip
+          pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
+        shell: bash
+
+      - name: Write clouds.yaml
+        run: |
+          mkdir -p ~/.config/openstack/
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
+        shell: bash
+
+      - name: Find CI clusters
+        run: |
+          . venv/bin/activate
+          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq)
+          echo "ci_clusters=${CI_CLUSTERS}" >> GITHUB_ENV
+        shell: bash
+      
+      - name: Delete clusters if control node not tagged with keep
+        run: |
+          . venv/bin/activate
+          for cluster_prefix in ${CI_CLUSTERS}
+          do
+            TAGS=$(openstack server show ${cluster_prefix}-control --column tags --format value)
+            if [[ $TAGS =~ "keep" ]]; then
+              echo "Skipping ${cluster_prefix} - control instance is tagged as keep"
+            else
+              yes | ./dev/delete-cluster.py ${cluster_prefix}
+            fi
+          done
+        shell: bash
diff --git a/.github/workflows/stackhpc.yml b/.github/workflows/stackhpc.yml
index 3dce03317..b08854adb 100644
--- a/.github/workflows/stackhpc.yml
+++ b/.github/workflows/stackhpc.yml
@@ -12,6 +12,8 @@ on:
       - '!docs/**'
       - '!README.md'
       - '!.gitignore'
+      - '!.github/workflows/'
+      - '.github/workflows/stackhpc'
   pull_request:
     paths:
       - '**'
@@ -20,6 +22,8 @@ on:
       - '!docs/**'
       - '!README.md'
       - '!.gitignore'
+      - '!.github/workflows/'
+      - '.github/workflows/stackhpc'
 jobs:
   openstack:
     name: openstack-ci