init democluster repo

.github/workflows/build-and-publish-image.yaml

@@ -0,0 +1,48 @@
+name: 'Build and Publish democluster to s3'
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build-image-in-lxd:
+    name: build-image-in-lxd
+    runs-on: self-hosted
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+      - name: Setup lxd controller
+        run: |
+          sudo snap install lxd --channel=latest/stable
+          sudo snap refresh lxd --channel=latest/stable
+          sudo lxd waitready
+          sudo lxd init --auto
+          sudo chmod a+wr /var/snap/lxd/common/lxd/unix.socket
+          lxc network set lxdbr0 ipv6.address none
+          sudo usermod -a -G lxd $USER
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+      - name: Setup poetry
+        uses: Gr1N/setup-poetry@v8
+      - name: Install image-factory
+        run: |
+          poetry env use "3.11"
+          poetry install
+      - name: Build democluster image
+        run: |
+          poetry run image-factory build democluster
+
+      - name: Sync democluster image to S3
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
+        run: |
+            sudo apt install awscli -y
+            aws s3 sync democluster/final \
+              s3://omnivector-public-assets/cloud-images/democluster/latest \
+              --acl public-read --follow-symlinks --delete 1> /dev/null

.github/workflows/ci.yaml

@@ -0,0 +1,52 @@
+name: 'Linter'
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  test:
+    name: "Lint code"
+    runs-on: "ubuntu-22.04"
+    strategy:
+      matrix:
+        python-version: ["3.11"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up Poetry
+        uses: Gr1N/setup-poetry@v8
+
+      - name: Install dependencies
+        run: |
+          poetry install
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+
+      - name: Cache Mypy dependencies
+        uses: actions/cache@v3
+        with:
+          path: ./.mypy_cache
+          key: ${{ runner.os }}-mypy-${{ hashFiles('.mypy_cache/CACHEDIR.TAG') }}
+
+      - name: Cache Ruff dependencies
+        uses: actions/cache@v3
+        with:
+          path: ./.ruff_cache
+          key: ${{ runner.os }}-ruff-${{ hashFiles('.ruff_cache/CACHEDIR.TAG') }}
+
+      - name: Run the linter checker
+        run: |
+          poetry run mypy image_factory --pretty
+          poetry run ruff check image_factory

.gitignore

@@ -0,0 +1,11 @@
+*.pyc
+__pycache__/**
+venv
+.venv
+
+output-*
+
+*.fd
+seeds-cloudimg.iso
+*.tar.gz
+*.img

README.md

@@ -1 +1,6 @@
 # democluster
+
+This project contains the democluster image producing codebase.
+
+###### Copyright
+Omnivector &copy; 2024 <admin@omnivector.soloutions>

democluster/CHANGELOG.rst

@@ -0,0 +1,19 @@
+=========
+Changelog
+=========
+
+Tracking of all notable changes to the Demo Cluster image.
+
+Unreleased
+----------
+
+0.2.0 - 2023-09-20
+------------------
+
+- Patch the demo cluster image to use the Jobbergate Agent instead of the Cluster Agent.
+- Patch the *deploy-democluster.sh* script to also set up the Jobbergate Agent instead of the Cluster Agent.
+
+0.1.0 - 2023-08-29
+------------------
+
+- First implementation of the demo cluster image.

democluster/Makefile

@@ -0,0 +1,55 @@
+PACKER ?= packer
+PACKER_LOG ?= 1
+
+
+.PHONY: clean-lxd
+clean-lxd:
+	./../scripts/clean_lxd.sh democluster
+
+.PHONY: clean
+clean:
+	sudo rm -rf output-*
+	rm -f *.fd
+	rm -f seeds-cloudimg.iso
+	rm -f *.img
+
+.PHONY: check-deps
+check-deps: ## Check deps needed to build the image
+	dpkg -s qemu-system wget libnbd-bin nbdkit fuse2fs cloud-image-utils wget > /dev/null
+
+.PHONY: init
+init: ## Run packer init .
+	${PACKER} init .
+
+.PHONY: stage0
+stage0: check-deps init ## Build democluster stage 0
+	PACKER_LOG=${PACKER_LOG} ${PACKER} build -only=stage0.* .
+
+.PHONY: final
+final: check-deps init ## Build democluster img file for use with multipass
+	PACKER_LOG=${PACKER_LOG} ${PACKER} build -only=final.* .
+
+.PHONY: all
+all: check-deps clean init stage0 final ## Build all democluster from the ground up
+
+.PHONY: delete-democluster-local
+delete-democluster-local: ## Delete the local democluster.
+	multipass delete democluster
+	multipass purge
+
+.PHONY: deploy-democluster-local 
+deploy-democluster-local: delete-democluster-local ## Deploy democluster from locally built image.
+	./helpers/deploy_local_democluster.sh $(CLIENT_ID) $(CLIENT_SECRET)
+
+.PHONY: help
+help:
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+
+# SETTINGS
+# Use one shell for all commands in a target recipe
+.ONESHELL:
+# Set default goal
+.DEFAULT_GOAL := help
+# Use bash shell in Make instead of sh
+SHELL := /bin/bash

democluster/democluster.pkr.hcl

@@ -0,0 +1,126 @@
+packer {
+  required_version = ">= 1.7.0"
+  required_plugins {
+    qemu = {
+      version = "~> 1.0"
+      source  = "github.com/hashicorp/qemu"
+    }
+  }
+}
+
+locals {
+  qemu_arch = {
+    "amd64" = "x86_64"
+    "arm64" = "aarch64"
+  }
+  uefi_imp = {
+    "amd64" = "OVMF"
+    "arm64" = "AAVMF"
+  }
+  qemu_machine = {
+    "amd64" = "ubuntu,accel=kvm"
+    "arm64" = "virt"
+  }
+  qemu_cpu = {
+    "amd64" = "host"
+    "arm64" = "cortex-a57"
+  }
+}
+
+variable "ubuntu_series" {
+  type        = string
+  default     = "jammy"
+  description = "The codename of the Ubuntu series to build."
+}
+
+variable "architecture" {
+  type        = string
+  default     = "amd64"
+  description = "The architecture to build the image for (amd64 or arm64)"
+}
+
+variable "http_directory" {
+  type    = string
+  default = "http"
+}
+
+source "null" "dependencies" {
+  communicator = "none"
+}
+
+source "null" "final" {
+  communicator = "none"
+}
+
+source "qemu" "stage0" {
+  boot_wait      = "2s"
+  cpus           = 4
+  disk_image     = true
+  disk_size      = "5G"
+  format         = "qcow2"
+  headless       = true
+  http_directory = var.http_directory
+  iso_checksum   = "file:https://cloud-images.ubuntu.com/${var.ubuntu_series}/current/SHA256SUMS"
+  iso_url        = "https://cloud-images.ubuntu.com/${var.ubuntu_series}/current/${var.ubuntu_series}-server-cloudimg-${var.architecture}.img"
+  memory         = 4096
+  qemu_binary    = "qemu-system-${lookup(local.qemu_arch, var.architecture, "")}"
+  qemuargs = [
+    ["-machine", "${lookup(local.qemu_machine, var.architecture, "")}"],
+    ["-cpu", "${lookup(local.qemu_cpu, var.architecture, "")}"],
+    ["-serial", "stdio"],
+    ["-device", "virtio-gpu-pci"],
+    ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/${lookup(local.uefi_imp, var.architecture, "")}/${lookup(local.uefi_imp, var.architecture, "")}_CODE.fd"],
+    ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=${lookup(local.uefi_imp, var.architecture, "")}_VARS.fd"],
+    ["-drive", "file=seeds-cloudimg.iso,format=raw"],
+    ["-drive", "file=output-stage0/packer-stage0,format=qcow2"]
+  ]
+  shutdown_command = "sudo -S shutdown -P now"
+  ssh_password     = "ubuntu"
+  ssh_username     = "ubuntu"
+  ssh_timeout      = "20m"
+}
+
+build {
+  name    = "stage0.deps"
+  sources = ["source.null.dependencies"]
+
+  provisioner "shell-local" {
+    inline = [
+      "cp /usr/share/${lookup(local.uefi_imp, var.architecture, "")}/${lookup(local.uefi_imp, var.architecture, "")}_VARS.fd ${lookup(local.uefi_imp, var.architecture, "")}_VARS.fd",
+      "cloud-localds seeds-cloudimg.iso user-data meta-data"
+    ]
+    inline_shebang = "/bin/bash -e"
+  }
+}
+
+build {
+  name    = "stage0.image"
+  sources = ["source.qemu.stage0"]
+
+  provisioner "shell" {
+    valid_exit_codes = [
+      "0",
+      "1",
+      "2"
+    ]
+    inline = [
+      "bash -c 'sleep 60'",
+      "bash -c 'python3 -W ignore /usr/bin/cloud-init status --wait'",
+      "bash -c 'sudo cloud-init clean --logs'",
+    ]
+  }
+}
+
+build {
+  name    = "final"
+  sources = ["source.null.final"]
+
+  post-processor "shell-local" {
+    inline = [
+        "mkdir -p final",
+        "cp output-stage0/packer-stage0 final/democluster.img",
+        "rm -rf output-stage0/",
+        "chmod -R 777 final",
+    ]
+  }
+}

democluster/helpers/deploy_local_democluster.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+CLIENT_ID=$1
+CLIENT_SECRET=$2
+
+cat <<EOF > /tmp/cloud-init.yaml
+#cloud-config
+runcmd:
+  - sed -i "s|@HEADNODE_HOSTNAME@|\$(hostname)|g" /etc/slurm/slurmdbd.conf
+  - sed -i "s|@HEADNODE_ADDRESS@|\$(hostname -I | awk '{print \$1}')|g" /etc/slurm/slurm.conf
+  - sed -i "s|@HEADNODE_HOSTNAME@|\$(hostname)|g" /etc/slurm/slurm.conf
+  - |
+    cpu_info=\$(lscpu -J | jq)
+
+    CPUs=\$(echo \$cpu_info | jq -r '.lscpu | .[] | select(.field == "CPU(s):") | .data')
+    sed -i "s|@CPUs@|\$CPUs|g" /etc/slurm/slurm.conf
+
+    THREADS_PER_CORE=\$(echo \$cpu_info | jq -r '.lscpu | .[] | select(.field == "Thread(s) per core:") | .data')
+    sed -i "s|@THREADS_PER_CORE@|\$THREADS_PER_CORE|g" /etc/slurm/slurm.conf
+
+    CORES_PER_SOCKET=\$(echo \$cpu_info | jq -r '.lscpu | .[] | select(.field == "Core(s) per socket:") | .data')
+    sed -i "s|@CORES_PER_SOCKET@|\$CORES_PER_SOCKET|g" /etc/slurm/slurm.conf
+
+    SOCKETS=\$(echo \$cpu_info | jq -r '.lscpu | .[] | select(.field == "Socket(s):") | .data')
+    sed -i "s|@SOCKETS@|\$SOCKETS|g" /etc/slurm/slurm.conf
+
+    REAL_MEMORY=\$(free -m | grep -oP '\\d+' | head -n 1)
+    sed -i "s|@REAL_MEMORY@|\$REAL_MEMORY|g" /etc/slurm/slurm.conf
+  - |
+    sed -i "s|@CLIENT_ID@|$CLIENT_ID|g" /srv/jobbergate-agent-venv/.env
+    sed -i "s|@CLIENT_SECRET@|$CLIENT_SECRET|g" /srv/jobbergate-agent-venv/.env
+  - |
+  - systemctl start slurmrestd
+  - systemctl restart slurmdbd
+  - systemctl restart slurmd
+  - sleep 30
+  - systemctl restart slurmctld
+  - scontrol update NodeName=\$(hostname) State=RESUME
+  - systemctl start jobbergate-agent
+EOF
+
+mkdir -p $HOME/democluster
+
+cat /tmp/cloud-init.yaml | multipass launch -c$(nproc) \
+-m4GB \
+--mount $HOME/democluster:/home/ubuntu/democluster \
+-ndemocluster \
+file://`pwd`/democluster.img \
+--cloud-init -
+
+rm -f /tmp/cloud-init.yaml

democluster/meta-data

@@ -0,0 +1,2 @@
+instance-id: iid-local01
+local-hostname: democluster