Remove slurm_openstack_tools collection (#537)

* remove slurm_openstack_tools collection

* bump CI images

* remove unused pytest role files

* remove unused reboot validation

* rename pytools to slurm_tools

* tidy slurm_tools

* fix slurm stats play

ansible/.gitignore

@@ -76,3 +76,7 @@ roles/*
 !roles/pulp_site/**
 !roles/doca/
 !roles/doca/**
+!roles/slurm_stats/
+!roles/slurm_stats/**
+!roles/pytools/
+!roles/pytools/**

ansible/monitoring.yml

@@ -17,14 +17,9 @@
 - name: Setup slurm stats
   hosts: slurm_stats
   tags: slurm_stats
-  collections:
-    - stackhpc.slurm_openstack_tools
   tasks:
     - include_role:
-        name: slurm-stats
-        apply:
-          # Collection currently requires root for all tasks.
-          become: true
+        name: slurm_stats
 
 - name: Deploy filebeat
   hosts: filebeat

ansible/roles/slurm_stats/README.md

@@ -0,0 +1,33 @@
+stackhpc.slurm_openstack_tools.slurm-stats
+==========================================
+
+Configures slurm-stats from https://github.com/stackhpc/slurm-openstack-tools.git which
+transforms sacct output into a form that is more amenable for importing into elasticsearch/loki.
+
+Requirements
+------------
+
+Role Variables
+--------------
+
+See `defaults/main.yml`.
+
+Dependencies
+------------
+
+Example Playbook
+----------------
+
+    - hosts: compute
+      tasks:
+        - import_role:
+            name: stackhpc.slurm_openstack_tools.slurm-stats
+
+
+License
+-------
+
+Apache-2.0
+
+Author Information
+------------------

ansible/roles/slurm_stats/defaults/main.yml

@@ -0,0 +1,13 @@
+---
+####################
+# log rotate options
+####################
+
+# These options affect the contents of the log-rotate file.
+# See: man logrotate
+
+# Log files are rotated count times before being removed
+slurm_stats_log_rotate_content_rotate: 7
+
+# How frequently are the log files rotated. Can be one of daily, monthly, ...
+slurm_stats_log_rotate_content_frequency: daily

ansible/roles/slurm_stats/tasks/main.yml

@@ -0,0 +1,34 @@
+---
+
+- name: Setup slurm tools
+  include_role:
+    name: slurm_tools
+
+- name: Create a directory to house the log files
+  file:
+    state: directory
+    path: /var/log/slurm-stats
+  become: true
+
+- name: Create cron job
+  cron:
+    name: Generate slurm stats
+    minute: "*/5"
+    user: root
+    # NOTE: lasttimestamp is stored at /root/lasttimestamp
+    job: "TZ=UTC /opt/slurm-tools/bin/slurm-stats >> /var/log/slurm-stats/finished_jobs.json"
+    cron_file: slurm-stats
+  become: true
+
+- name: Setup log rotate
+  copy:
+    content: |
+      # WARNING: This file is managed by ansible, do not modify.
+      /var/log/slurm-stats/finished_jobs.json {
+              {{ slurm_stats_log_rotate_content_frequency }}
+              rotate {{ slurm_stats_log_rotate_content_rotate }}
+              compress
+              delaycompress
+      }
+    dest: /etc/logrotate.d/slurm-stats
+  become: true

ansible/roles/slurm_tools/.travis.yml

@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/

ansible/roles/slurm_tools/README.md

@@ -0,0 +1,12 @@
+slurm_tools
+=========
+
+Install python-based tools from https://github.com/stackhpc/slurm-openstack-tools.git into `/opt/slurm-tools/bin/`.
+
+Role Variables
+--------------
+
+- `pytools_editable`: Optional bool. Whether to install the package using `pip`'s
+  editable mode (installing source to `/opt/slurm-tools/src`). Default `false`.
+- `pytools_gitref`: Optional. Git branch/tag/commit etc to install. Default `master`.
+- `pytools_user`: Optional user to install as. Default `root`.

ansible/roles/slurm_tools/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+pytools_editable: false
+pytools_gitref: master
+pytools_user: root

ansible/roles/slurm_tools/tasks/main.yml

@@ -0,0 +1,33 @@
+---
+- name: install python3
+  package:
+    name: python3,git
+  become: true
+
+- name: Create virtualenv directory
+  file:
+    path: /opt/slurm-tools
+    owner: "{{ pytools_user }}"
+    group: "{{ pytools_user }}"
+    state: directory
+  become: true
+
+- block:
+  - name: Upgrade pip
+    # This needs to a separate step so that we use the updated version
+    # to install the packages below.
+    pip:
+      name: pip
+
+  - name: Create virtualenv
+    pip:
+      name: "git+https://github.com/stackhpc/slurm-openstack-tools.git@{{ pytools_gitref }}#egg=slurm_openstack_tools"
+      editable: "{{ pytools_editable }}"
+
+  module_defaults:
+    ansible.builtin.pip:
+      virtualenv: /opt/slurm-tools
+      virtualenv_command: python3 -m venv
+      state: latest
+  become: true
+  become_user: "{{ pytools_user }}"

ansible/slurm.yml

@@ -9,16 +9,6 @@
     - include_role:
         name:  mysql
 
-- name: Setup slurm-driven reimage
-  hosts: rebuild
-  become: yes
-  tags:
-    - rebuild
-    - openhpc
-  tasks:
-    - import_role:
-        name: stackhpc.slurm_openstack_tools.rebuild
-
 - name: Setup slurm
   hosts: openhpc
   become: yes

ansible/validate.yml

@@ -40,16 +40,6 @@
         tasks_from: validate.yml
       tags: validate
 
-- name: Validate rebuild configuration
-  hosts: rebuild
-  gather_facts: false
-  tags: rebuild
-  tasks:
-    - import_role:
-        name: stackhpc.slurm_openstack_tools.rebuild
-        tasks_from: validate.yml
-      tags: validate
-
 - name: Validate openondemand configuration
   hosts:
     - openondemand

environments/.stackhpc/terraform/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250114-1627-bccc88b5",
-        "RL9": "openhpc-RL9-250114-1626-bccc88b5"
+        "RL8": "openhpc-RL8-250115-1510-99f67c6d",
+        "RL9": "openhpc-RL9-250115-1510-99f67c6d"
     }
 }

requirements.yml

@@ -28,9 +28,6 @@ collections:
     version: 1.16.2
   - name: community.grafana
     version: 1.5.4
-  - name: https://github.com/stackhpc/ansible_collection_slurm_openstack_tools
-    type: git
-    version: v0.2.0
   - name: ansible.posix
     version: 1.5.4
   - name: ansible.netcommon