Update to Debian 12 && Fix VM compatibility issues

- TH_SRC switched to "archive" to avoid update-removals
  + Move iso retrieval stuff to Makefile
- Switch from packaged salt to pip-based install
- Split apt to support different OS versions
- Fix compatibility issues with Debian 12
  + Ensure /dev/shm is mounted
- Add coreutils to cicd workflow
- Add more retries to early boot for vbox test
- Update testing to work on changes to github infrastructure

.github/workflows/cicd.yml

@@ -68,16 +68,13 @@ jobs:
       - name: Install Dependencies
         run: |
             sudo apt-get update
-            sudo apt-get install virtualbox libarchive-tools syslinux xorriso isolinux python3-distro
+            sudo apt-get install virtualbox libarchive-tools syslinux xorriso isolinux python3-distro coreutils
 
       # 1. Build a "teckhost" iso from upstream release
       - name: Build Teckhost ISO
         id: build_iso
         run: make teckhost-sda.iso
         env:
-          # current stable
-          TH_SRC: https://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/current/amd64/iso-cd/firmware-11.7.0-amd64-netinst.iso
-          TH_CKSUM: "029500297f14bd4f6650fa4aa991c96027b8d5cd9cc91b38722ee2b914612e851fa81d60f4e7ad739565a83128a98461368fc95defc01ec7d66c62a32ca15bf9"
           THT_GRUBTEST: TEMPLATE_METHOD=lvm BS_pillar_root=test/pillar TH_SALTGPG=https://raw.githubusercontent.com/MTecknology/teckhost/master/test/pillar/skeys.gpg BS_gitfs_pillar_base=master BS_gitfs_base=${{ github.sha }}
 
       - name: Save ISO (teckhost-sda.iso)
@@ -92,15 +89,21 @@ jobs:
 
     strategy:
       matrix:
-        boot: [efi, bios]
+        #boot: [efi, bios]
+        boot: [efi]
+        #os: [debian11, debian12]
+        os: [debian12]
 
-    # virtualbox can only run on some versions (10.5, 12) of macosx runners
     runs-on: macos-12
     steps:
       - uses: actions/checkout@v3
 
       - name: Install Dependencies
-        run: pip3 install pytest-testinfra distro
+        run: |
+            brew install coreutils
+            #brew install --cask virtualbox
+            pip3 install pytest-testinfra distro
+            #sh ./test/vbox_extpack
 
       - name: Pull ISO (teckhost-sda.iso)
         uses: actions/download-artifact@v3
@@ -116,6 +119,10 @@ jobs:
           TH_SHOTS: testpc1-${{ matrix.boot }}
           TH_BOOT: ${{ matrix.boot }}
 
+      - name: (on failure) Package Screenshots
+        if: failure() && steps.install_os.outcome == 'failure'
+        run: "tar -vcf testpc1-${{ matrix.boot }}.tar *.png"
+
       - name: (on failure) Upload Screenshots
         if: failure() && steps.install_os.outcome == 'failure'
         uses: actions/upload-artifact@v3

.github/workflows/release.yml

@@ -38,16 +38,12 @@ jobs:
       - name: Install Build Dependencies
         run: |
             sudo apt-get update
-            sudo apt-get install libarchive-tools syslinux xorriso isolinux
+            sudo apt-get install libarchive-tools syslinux xorriso isolinux coreutils
 
       # 2. Build "Production" ISO (teckhost*.iso)
       - name: Build Teckhost ISO
         id: build_iso
         run: make teckhost.iso teckhost-sda.iso teckhost-nvme0n1.iso
-        env:
-          # current stable
-          TH_SRC: https://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/current/amd64/iso-cd/firmware-11.7.0-amd64-netinst.iso
-          TH_CKSUM: "029500297f14bd4f6650fa4aa991c96027b8d5cd9cc91b38722ee2b914612e851fa81d60f4e7ad739565a83128a98461368fc95defc01ec7d66c62a32ca15bf9"
 
       # 3. Publish Release w/ Artifacts
       - name: Create Release

Makefile

@@ -6,37 +6,54 @@
 export WORKSPACE ?= $(abspath $(PWD)/)
 export GRUB_EXTRA ?= hostname=testpc1
 
+# Version Table
+debian12_src ?= https://cdimage.debian.org/cdimage/archive/12.1.0/amd64/iso-cd/debian-12.1.0-amd64-netinst.iso
+debian12_sha ?= 9f181ae12b25840a508786b1756c6352a0e58484998669288c4eec2ab16b8559
+
 
 ##
 # ISO
 ##
 
-# Intended for production use (assumes nvme)
-teckhost.iso: iso/preseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg
+# Intended for production use
+teckhost.iso: upstream_debian12.iso iso/preseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg
 	./iso/build_iso \
 	    -s iso/preseed.cfg \
+	    -i upstream_debian12.iso \
 	    -o teckhost.iso \
 	    -x "$(GRUB_EXTRA)" \
 	    -f iso/grub-bios.cfg -g iso/grub-efi.cfg
 
 # Intended for use with automated testing
-teckhost-%.iso: testseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg
+teckhost-%.iso: upstream_debian12.iso testseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg
 	./iso/build_iso \
 	    -s testseed.cfg \
+	    -i upstream_debian12.iso \
 	    -o "$@" \
 	    -d "/dev/$*" \
 	    -x "$(GRUB_EXTRA)" \
 	    -f iso/grub-bios.cfg -g iso/grub-efi.cfg
 
 # Intended for local developmnt with virtualbox
-teckhost-local.iso: testseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg
+teckhost-local.iso: upstream_debian12.iso testseed.cfg iso/grub-bios.cfg iso/grub-efi.cfg
 	./iso/build_iso \
 	    -s testseed.cfg \
+	    -i upstream_debian12.iso \
 	    -o teckhost-local.iso \
 	    -d /dev/sda \
 	    -x "hostname=devpc1 BS_devdir=/srv" \
 	    -f iso/grub-bios.cfg -g iso/grub-efi.cfg
 
+# Grab an upstream ISO and validate checksum
+upstream_%.iso:
+	# Copy iso from parent directory or download fresh copy
+	cp "../$($*_sha).iso" ./ || wget --quiet -O "$($*_sha).iso" "$($*_src)"
+	# Verify checksum of pristine iso
+	echo "$($*_sha)  $($*_sha).iso" | sha256sum -c
+	# Move into location to verify success
+	mv "$($*_sha).iso" "upstream_$*.iso"
+
+
 ##
 # Preeseed
 ##

bootstrap

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 ##
 # A quick/simple script to get masterless salt deployed and configured.
 # See help text (./bootstrap -h) and README.rst for requirements.
@@ -9,27 +10,34 @@
 #
 # BS_*: This script will dump all "BS_*" environment variables into a configuration
 #       file for salt-minion; this provides a way to mangle teckhost.conf.
+#
+# Critical Order:
+#   1. Prepare apt
+#   2. Install apt/salt dependencies,
+#   3. Add salt repo
+#   4. Install salt
+#   5. Chown salt
 ##
 
 main() {
 	parse_options "$@"
 
-	# Prep
+	# Pre-flight
 	safety_checks
 	lock acquire "$0" || die 'Unable to acquire lock'
+	pristine_apt || die 'Failed to set pristine apt configuration'
 	mkdir -p /etc/salt/minion.d
+	echo 'master: invalid.tld' >/etc/salt/minion.d/teckhost.conf
 
-	# Patch gai.conf
+	# Hack: IPv6 is often incorrectly implemented and first boot is very touchy
 	echo 'precedence ::ffff:0:0/96  100' >>/etc/gai.conf
+	echo 'Acquire::ForceIPv4 "true";' >/etc/apt/apt.conf.d/99force-ipv4
 
-	# Install Masterless Salt
-	configure_apt || die 'Failed to configure apt'
-	apt-get -y install gpg python3-pygit2 wget || die 'Failed to install dependencies'
-	configure_minion || die 'Failed to install salt-minion'
+	# Install and configure salt-minion (solo)
+	install_salt || die 'Failed to install salt-minion service'
+	configure_minion || die 'Failed to configure salt-minion'
 	deploy_gpgkeys || die 'Failed to unpack GPG keys'
-	#TODO: This is TEMPORARY hack because we need >= 3004.0
-	apt-get install -y salt-minion=3004.1+dfsg-2 salt-common=3004.1+dfsg-2 || die 'Failed to install salt-minion'
-	#apt-get install -y salt-minion || die 'Failed to install salt-minion'
+	chown -R root:root /etc/salt/gpgkeys
 
 	# Run Highstate and Configure System
 	run_highstate || die 'Provisioning process (highstate) failed'
@@ -86,7 +94,7 @@ safety_checks() {
 }
 
 # Ensure a clean apt state prior to salt management
-configure_apt() {
+pristine_apt() {
 	# Find $OSCODENAME
 	# Seems excessive when only one path is likely, but who knows how this might get used
 	if [[ -n "$OSCODENAME" ]]; then
@@ -105,13 +113,13 @@ configure_apt() {
 	rm -rf /etc/apt/sources.list*
 	mkdir /etc/apt/sources.list.d
 	cat >/etc/apt/sources.list <<-EOF
-		deb http://deb.debian.org/debian $OSCODENAME main contrib non-free
-		deb http://security.debian.org/debian-security $OSCODENAME-security main contrib non-free
-		deb http://deb.debian.org/debian $OSCODENAME-updates main contrib non-free
+		deb http://deb.debian.org/debian $OSCODENAME main contrib non-free non-free-firmware
+		deb http://security.debian.org/debian-security $OSCODENAME-security main contrib non-free non-free-firmware
+		deb http://deb.debian.org/debian $OSCODENAME-updates main contrib non-free non-free-firmware
 
 		# Newer Packages (use with extreme caution)
-		deb http://deb.debian.org/debian testing main non-free contrib
-		deb http://deb.debian.org/debian sid main non-free contrib
+		deb http://deb.debian.org/debian testing main non-free non-free-firmware contrib
+		deb http://deb.debian.org/debian sid main non-free non-free-firmware contrib
 		EOF
 	cat >/etc/apt/preferences.d/pinning <<-EOF
 		Package: *
@@ -130,17 +138,40 @@ configure_apt() {
 		Pin: release a=unstable
 		Pin-Priority: 300
 		EOF
+
+	# Update package cache
 	apt-get update
 }
 
+# Install the salt master and get vendor-garbage configured correctly
+install_salt() {
+	# Dependencies
+	apt-get update || return 1
+	apt-get install -y debconf-utils wget python3-venv build-essential git || return 1
+
+        # Application
+        python3 -m venv /opt/salt
+        /opt/salt/bin/pip3 install cryptography pygit2 'Jinja2<3.1' salt==3006.6
+
+        # Directory structure
+        mkdir -p /etc/salt/minion.d
+        mkdir -p /etc/salt/pki/minion
+}
+
 # Run a highstate
 run_highstate() {
+	# Ensure /dev/shm is mounted
+	if ! mountpoint -q /dev/shm; then
+		test -d /dev/shm || mkdir /dev/shm
+		mount -t tmpfs none /dev/shm
+	fi
+
 	# This is an ugly hack because of some networking hiccups during some deployments.
-	if ! salt-call --local -l debug state.highstate; then
+	if ! /opt/salt/bin/salt-call --local -l info state.highstate; then
 		log "$WARN" 'FIRST HIGHSTATE FAILED; Sleeping a few minutes before retrying.'
 		sleep 240
 		# Less verbosity to help with information gathering
-		salt-call --local -l quiet --state-verbose=false state.highstate
+		/opt/salt/bin/salt-call --local -l quiet --state-verbose=false state.highstate
 	fi
 }
 

iso/build_iso

@@ -28,7 +28,6 @@ main() {
 # Read options into environment
 parse_options() {
 	# Defaults
-	export TH_CKSUM="$TH_CKSUM"
 	export TH_SRC="${TH_SRC:-./debian-netinst.iso}"
 	export TH_DST="${TH_DST:-./teckhost.iso}"
 	export TH_SEED="${TH_SEED:-preseed.cfg}"
@@ -40,9 +39,8 @@ parse_options() {
 	export THT_DEVICE="${THT_DEVICE:-/dev/nvme0n1}"
 	export LOG_LEVEL="${LOG_LEVEL:-1}"
 
-	while getopts 'c:i:o:s:f:g:kd:b:x:l:h' OPT; do
+	while getopts 'i:o:s:f:g:kd:b:x:l:h' OPT; do
 		case "$OPT" in
-			c) TH_CKSUM="$OPTARG";;
 			i) TH_SRC="$OPTARG";;
 			o) TH_DST="$OPTARG";;
 			s) TH_SEED="$OPTARG";;
@@ -57,6 +55,9 @@ parse_options() {
 			*) die "Unexpected argument provided: '$OPT'";;
 		esac
 	done
+
+	# Enable real debugging
+	[ "${LOG_LEVEL:-1}" -lt 1 ] && set -x
 }
 
 # Show help text (an explanation of options)
@@ -68,7 +69,6 @@ show_help() {
 	Usage: build_iso [options]
 
 	Options:
-	  -c X${t}Checksum of resource
 	  -i X${t}Pristine ISO to be modified (can be http)
 	  -o X${t}Output ISO
 	  -s X${t}Preseed template to embed into ISO
@@ -82,7 +82,6 @@ show_help() {
 	  -h${t}Print this help text and exit
 
 	Defaults (can be set as environment variables):
-	  TH_CKSUM${t}<empty>
 	  TH_SRC${t}./debian-netinst.iso
 	  TH_DST${t}./teckhost.iso
 	  TH_SEED${t}./preseed.cfg
@@ -93,21 +92,15 @@ show_help() {
 	  THT_GRUBTEST${t}TEMPLATE_METHOD=lvm BS_pillar_root=test/pillar TH_SALTGPG=https://raw.githubusercontent.com/MTecknology/teckhost/master/test/pillar/skeys.gpg
 	  THT_GRUBTXTRA${t}""
 	  LOG_LEVEL${t}1  (info)
-
-	Tell build_iso to pull a remote iso and store in /tmp/\$TH_CKSUM.iso:
-	  export \\
-	      TH_SRC=https://cdimage.debian.org/cdimage/unofficial/non-free/cd-including-firmware/current/amd64/iso-cd/firmware-11.3.0-amd64-netinst.iso \\
-	      TH_CKSUM=eba7ce7823681a610f9f23d6468976517ed92b6b90acec4ac55df62b0a090050bba0145ef5c07f544b92569cd10e9572f4e9f7c3415b3323abffa51cd7c5d4f4
-	  ./build_iso -o ~/teckhost.iso
 	EOF
 }
 
 safety_checks() {
-	log "$DEBUG" 'Running sanity checks'
+	log "$INFO" 'Running sanity checks'
 
 	# Check for required commands
 	for cmd in 'bsdtar' 'syslinux' 'xorriso'; do
-		command_present "$cmd" || die 'Missing dependencies, must have: bsdtar syslinux xorriso'
+		command_present "$cmd" || die 'Missing dependencies, must have: bsdtar (libarchive-tools) syslinux xorriso'
 	done
 
 	# Verify provided values point at files
@@ -119,48 +112,16 @@ safety_checks() {
 }
 
 unpack_source() {
-	# If source is http, grab/cache resource
-	if [[ "$TH_SRC" = http* ]]; then
-		_get_websource || die 'Failed to grab resource'
-	fi
-
 	# Verify source exists
 	[[ ! -f "$TH_SRC" ]] && die "Source file ($TH_SRC) does not exist"
 
-	# Verify checksum of source
-	if [[ -n "$TH_CKSUM" ]]; then
-		sha512sum "$TH_SRC" | cut -d' ' -f1 | grep -q "$TH_CKSUM" || \
-			die "cksum($TH_SRC) != $TH_CKSUM"
-	fi
-
 	# Actually unpack source
 	bsdtar -C "$TH_TEMP" -xf "$TH_SRC"
 
 	# Make it editable (must be reverted before regen)
 	chmod -R +w "$TH_TEMP"
 }
 
-# Pull/cache a copy from the web and swap variables once verified
-_get_websource() {
-	command_present 'wget' || die 'wget is required to get a web source'
-
-	# Local cache location
-	iso="/tmp/$TH_CKSUM.iso"
-
-	# If cache file exists, validate and return
-	if [[ -f "$iso" ]]; then
-		if sha512sum "$iso" | cut -d' ' -f1 | grep -q "$TH_CKSUM"; then
-			log "$DEBUG" "Valid cache for $TH_SRC"
-			TH_SRC="$iso"
-			return 0
-		fi
-		log "$WARN" "Removing invalid cache at $iso"
-		rm -f "$iso"
-	fi
-	# Grab resource and store in cache location
-	wget -qO "$iso" "$TH_SRC" && TH_SRC="$iso"
-}
-
 # Insert an auto-only grub boot config
 inject_grubconfig() {
 	log "$DEBUG" 'Copying grub config'
@@ -195,7 +156,7 @@ inject_preseed() {
 
 # Build the modified ISO
 build_iso() {
-	log "$DEBUG" "Bulding ISO at $TH_DST"
+	log "$INFO" "Bulding ISO at $TH_DST"
 	# Some weird requirement; things break without it
 	chmod -R -w "$TH_TEMP"
 	# command partially copied from iso in .disk/isofs

states/apt/init.sls

@@ -3,7 +3,7 @@ include:
 
 /etc/apt/sources.list:
   file.managed:
-    - source: salt://apt/sources.list
+    - source: salt://apt/sources.list_{{ salt.grains.get('osfinger') }}
     - template: jinja
     - require:
       - file: /etc/apt/preferences.d/pinning

states/apt/sources.list_Debian-12

@@ -0,0 +1,8 @@
+{% if salt.grains.get('osrelease') not in ['testing'] -%}
+deb http://deb.debian.org/debian/ {{ salt.grains.get('oscodename') }} main contrib non-free non-free-firmware
+deb http://security.debian.org/debian-security {{ salt.grains.get('oscodename') }}-security main contrib non-free non-free-firmware
+deb http://deb.debian.org/debian/ {{ salt.grains.get('oscodename') }}-updates main contrib non-free non-free-firmware
+deb http://deb.debian.org/debian/ {{ salt.grains.get('oscodename') }}-backports main contrib non-free non-free-firmware
+{% endif -%}
+deb http://deb.debian.org/debian/ testing main contrib non-free non-free-firmware
+deb http://deb.debian.org/debian/ sid main contrib non-free non-free-firmware