From 6cc2e3ed53708bc07da7932adc0326faac9c670f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 14 Oct 2023 21:09:25 +0200
Subject: [PATCH 01/67] first version of bot/test.sh script

---
 bot/test.sh | 224 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 224 insertions(+)
 create mode 100755 bot/test.sh

diff --git a/bot/test.sh b/bot/test.sh
new file mode 100755
index 0000000000..6d217caf92
--- /dev/null
+++ b/bot/test.sh
@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+#
+# script to run tests or the test suite for the whole EESSI software layer or
+# just what has been built in a job. Intended use is that it is called
+# at the end of a (batch) job running on a compute node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# ASSUMPTIONs:
+# + assumption for the build step (as run through bot/build.sh which is provided
+#   in this repository too)
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files referenced in job.cfg
+# + assumptions for the test step
+#  - temporary storage is still available
+#    example
+#    Using /localscratch/9640860/NESSI/eessi.x765Dd8mFh as tmp directory (to resume session add '--resume /localscratch/9640860/NESSI/eessi.x765Dd8mFh').
+#  - run test-suite.sh inside build container using tmp storage from build step
+#    plus possibly additional settings (repo, etc.)
+#  - needed setup steps may be similar to bot/inspect.sh (PR#317)
+
+# stop as soon as something fails
+set -e
+
+# source utils.sh and cfg_files.sh
+source scripts/utils.sh
+source scripts/cfg_files.sh
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+HOST_ARCH=$(uname -m)
+
+# check if ${JOB_CFG_FILE} exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+fi
+echo "bot/test.sh: showing ${JOB_CFG_FILE} from software-layer side"
+cat ${JOB_CFG_FILE}
+
+echo "bot/test.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+cfg_load ${JOB_CFG_FILE}
+
+# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy
+HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
+HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
+echo "bot/test.sh: HTTP_PROXY='${HTTP_PROXY}'"
+
+# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy
+HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
+HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
+echo "bot/test.sh: HTTPS_PROXY='${HTTPS_PROXY}'"
+
+LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
+echo "bot/test.sh: LOCAL_TMP='${LOCAL_TMP}'"
+# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
+
+# check if path to copy build logs to is specified, so we can copy build logs for failing builds there
+BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir")
+echo "bot/test.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'"
+# if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container
+if [[ ! -z ${BUILD_LOGS_DIR} ]]; then
+    mkdir -p ${BUILD_LOGS_DIR}
+    if [[ -z ${SINGULARITY_BIND} ]]; then
+        export SINGULARITY_BIND="${BUILD_LOGS_DIR}"
+    else
+        export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}"
+    fi
+fi
+
+# check if path to directory on shared filesystem is specified,
+# and use it as location for source tarballs used by EasyBuild if so
+SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path")
+echo "bot/test.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'"
+# if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container
+if [[ ! -z ${SHARED_FS_PATH} ]]; then
+    mkdir -p ${SHARED_FS_PATH}
+    if [[ -z ${SINGULARITY_BIND} ]]; then
+        export SINGULARITY_BIND="${SHARED_FS_PATH}"
+    else
+        export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}"
+    fi
+fi
+
+SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
+echo "bot/test.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    # make sure that separate directories are used for different CPU families
+    SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH}
+    export SINGULARITY_CACHEDIR
+fi
+
+# try to determine tmp directory from build job
+RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -1 | awk '{print $2}')
+
+if [[ -z ${RESUME_DIR} ]]; then
+  echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
+  # replace any env variable in ${LOCAL_TMP} with its
+  #   current value (e.g., a value that is local to the job)
+  STORAGE=$(envsubst <<< ${LOCAL_TMP})
+  echo "'${STORAGE}'"
+
+  # make sure ${STORAGE} exists
+  mkdir -p ${STORAGE}
+
+  # make sure the base tmp storage is unique
+  JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX)
+  echo "bot/test.sh: created unique base tmp storage directory at ${JOB_STORAGE}"
+
+  RESUME_TGZ=${PWD}/previous_tmp/build_step/$(ls previous_tmp/build_step)
+  if [[ -z ${RESUME_TGZ} ]]; then
+    echo "bot/test.sh: no information about tmp directory and tarball of build step; --> giving up"
+    exit 2
+  fi
+fi
+
+# obtain list of modules to be loaded
+LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
+echo "bot/test.sh: LOAD_MODULES='${LOAD_MODULES}'"
+
+# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
+CONTAINER=$(cfg_get_value "repository" "container")
+export SINGULARITY_HOME="${PWD}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir"
+mkdir -p ${SINGULARITY_TMPDIR}
+
+# load modules if LOAD_MODULES is not empty
+if [[ ! -z ${LOAD_MODULES} ]]; then
+    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    do
+        echo "bot/test.sh: loading module '${mod}'"
+        module load ${mod}
+    done
+else
+    echo "bot/test.sh: no modules to be loaded"
+fi
+
+# determine repository to be used from entry .repository in ${JOB_CFG_FILE}
+REPOSITORY=$(cfg_get_value "repository" "repo_id")
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+echo "bot/test.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
+
+# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
+# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# next script (eessi_container.sh) makes use of it via sourcing init scripts
+# (e.g., init/eessi_defaults or init/minimal_eessi_env)
+export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
+
+# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
+# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
+# "source init/eessi_defaults" via sourcing init/minimal_eessi_env
+export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
+echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
+
+# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}
+# fallbacks:
+#  - ${CPU_TARGET} handed over from bot
+#  - left empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}}
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE
+echo "bot/test.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'"
+
+# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
+EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
+export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
+echo "bot/test.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
+
+# prepare arguments to eessi_container.sh common to build and tarball steps
+declare -a COMMON_ARGS=()
+COMMON_ARGS+=("--verbose")
+COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--mode" "run")
+[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
+[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}")
+
+# make sure to use the same parent dir for storing tarballs of tmp
+PREVIOUS_TMP_DIR=${PWD}/previous_tmp
+
+# prepare directory to store tarball of tmp for build step
+TARBALL_TMP_TEST_STEP_DIR=${PREVIOUS_TMP_DIR}/test_step
+mkdir -p ${TARBALL_TMP_TEST_STEP_DIR}
+
+# prepare arguments to eessi_container.sh specific to test step
+declare -a TEST_STEP_ARGS=()
+TEST_STEP_ARGS+=("--save" "${TARBALL_TMP_TEST_STEP_DIR}")
+
+if [[ -z ${RESUME_DIR} ]]; then
+  TEST_STEP_ARGS+=("--storage" "${STORAGE}")
+  TEST_STEP_ARGS+=("--resume" "${RESUME_TGZ}")
+else
+  TEST_STEP_ARGS+=("--resume" "${RESUME_DIR}")
+fi
+
+# prepare arguments to test_suite.sh (specific to test step)
+declare -a TEST_SUITE_ARGS=()
+# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
+#     TEST_SUITE_ARGS+=("--generic")
+# fi
+# [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
+# [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
+
+# create tmp file for output of build step
+test_outerr=$(mktemp test.outerr.XXXX)
+
+echo "Executing command to build software:"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
+echo "                     -- ./test_suite.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
+./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \
+                     -- ./test_suite.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
+
+exit 0

From 9a32dc629b34e4bbc91fda349198611a6a96a082 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 15 Oct 2023 20:29:14 +0200
Subject: [PATCH 02/67] scripts to run tests after a build job has finished

---
 run_tests.sh  |   4 ++
 test_suite.sh | 184 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 run_tests.sh
 create mode 100644 test_suite.sh

diff --git a/run_tests.sh b/run_tests.sh
new file mode 100644
index 0000000000..ccbc751f22
--- /dev/null
+++ b/run_tests.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+base_dir=$(dirname $(realpath $0))
+source ${base_dir}/init/eessi_defaults
+./run_in_compat_layer_env.sh ./test_suite.sh "$@"
diff --git a/test_suite.sh b/test_suite.sh
new file mode 100644
index 0000000000..fcaaa339d2
--- /dev/null
+++ b/test_suite.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+#
+# Run sanity check for all requested modules (and built dependencies)?
+# get ec from diff
+# set up EasyBuild
+# run `eb --sanity-only ec`
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo "  -g | --generic         -  instructs script to test for generic architecture target"
+  echo "  -h | --help            -  display this usage information"
+  echo "  -x | --http-proxy URL  -  provides URL for the environment variable http_proxy"
+  echo "  -y | --https-proxy URL -  provides URL for the environment variable https_proxy"
+}
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -g|--generic)
+      EASYBUILD_OPTARCH="GENERIC"
+      shift
+      ;;
+    -h|--help)
+      display_help  # Call your function
+      # no shifting needed here, we're done.
+      exit 0
+      ;;
+    -x|--http-proxy)
+      export http_proxy="$2"
+      shift 2
+      ;;
+    -y|--https-proxy)
+      export https_proxy="$2"
+      shift 2
+      ;;
+    --build-logs-dir)
+      export build_logs_dir="${2}"
+      shift 2
+      ;;
+    --shared-fs-path)
+      export shared_fs_path="${2}"
+      shift 2
+      ;;
+    -*|--*)
+      echo "Error: Unknown option: $1" >&2
+      exit 1
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+TOPDIR=$(dirname $(realpath $0))
+
+source $TOPDIR/scripts/utils.sh
+
+# honor $TMPDIR if it is already defined, use /tmp otherwise
+if [ -z $TMPDIR ]; then
+    export WORKDIR=/tmp/$USER
+else
+    export WORKDIR=$TMPDIR/$USER
+fi
+
+TMPDIR=$(mktemp -d)
+
+echo ">> Setting up environment..."
+
+source $TOPDIR/init/minimal_eessi_env
+
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
+else
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
+fi
+
+# make sure we're in Prefix environment by checking $SHELL
+if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
+    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
+else
+    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
+fi
+
+# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
+export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+
+DETECTION_PARAMETERS=''
+GENERIC=0
+EB='eb'
+if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
+    echo_yellow ">> GENERIC build/test requested, taking appropriate measures!"
+    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
+    GENERIC=1
+    EB='eb --optarch=GENERIC'
+fi
+
+echo ">> Determining software subdirectory to use for current build/test host..."
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
+
+# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+# $EESSI_SILENT - don't print any messages
+# $EESSI_BASIC_ENV - give a basic set of environment variables
+EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+
+if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
+    fatal_error "Failed to determine software subdirectory?!"
+elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
+else
+    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
+fi
+
+echo ">> Initializing Lmod..."
+source $EPREFIX/usr/share/Lmod/init/bash
+ml_version_out=$TMPDIR/ml.out
+ml --version &> $ml_version_out
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Found Lmod ${LMOD_VERSION}"
+else
+    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
+fi
+
+echo ">> Configuring EasyBuild..."
+source $TOPDIR/configure_easybuild
+
+echo ">> Setting up \$MODULEPATH..."
+# make sure no modules are loaded
+module --force purge
+# ignore current $MODULEPATH entirely
+module unuse $MODULEPATH
+module use $EASYBUILD_INSTALLPATH/modules/all
+if [[ -z ${MODULEPATH} ]]; then
+    fatal_error "Failed to set up \$MODULEPATH?!"
+else
+    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
+fi
+
+# assume there's only one diff file that corresponds to the PR patch file
+pr_diff=$(ls [0-9]*.diff | head -1)
+
+# "split" the file by prefixing each line belonging to the same file with the
+# same number
+split_file=$(awk '/^\+\+\+/{n++}{print n, "  ", $0 }' ${pr_diff})
+
+# determine which easystack files may have changed
+changed_es_files=$(echo "${split_file}" | grep '^[0-9 ]*+++ ./eessi.*.yml$' | egrep -v 'known-issues|missing')
+
+# process all changed easystackfiles
+for es_file_num in $(echo "${changed_es_files}" | cut -f1 -d' ')
+do
+    # determine added lines that do not contain a yaml comment only
+    added_lines=$(echo "${split_file}" | grep "${es_file_num}    + " | sed -e "s/^"${es_file_num}"    + //" | grep -v "^[ ]*#")
+    # determine easyconfigs
+    easyconfigs=$(echo "${added_lines}" | cut -f3 -d' ')
+    # get easystack file name
+    easystack_file=$(echo "${changed_es_files}" | grep "^${es_file_num}" | sed -e "s/^"${es_file_num}"    ... .\///")
+    echo -e "Processing easystack file ${easystack_file}...\n\n"
+
+    # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+    eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+    # load EasyBuild module
+    module load EasyBuild/${eb_version}
+
+    echo_green "All set, let's run sanity checks for installed packages..."
+
+    for easyconfig in ${easyconfigs};
+    do
+        echo "Running sanity check for '${easyconfig}'..."
+        eb --sanity-check-only ${easyconfig}
+    done
+done
+
+echo ">> Cleaning up ${TMPDIR}..."
+rm -r ${TMPDIR}

From 17cfd02779833bd5f2df96655f3dc19bb5f880e8 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 15 Oct 2023 20:30:44 +0200
Subject: [PATCH 03/67] small adjustments to test script

---
 bot/test.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bot/test.sh b/bot/test.sh
index 6d217caf92..89c1a6a8bf 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -206,9 +206,9 @@ fi
 
 # prepare arguments to test_suite.sh (specific to test step)
 declare -a TEST_SUITE_ARGS=()
-# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
-#     TEST_SUITE_ARGS+=("--generic")
-# fi
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
+    TEST_SUITE_ARGS+=("--generic")
+fi
 # [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
 # [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
 
@@ -217,8 +217,8 @@ test_outerr=$(mktemp test.outerr.XXXX)
 
 echo "Executing command to build software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
-echo "                     -- ./test_suite.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
+echo "                     -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \
-                     -- ./test_suite.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
+                     -- ./run_tests.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
 
 exit 0

From 0f3646ef0eec6a89aa1a8a5479c8040264af5af3 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 5 Feb 2024 17:28:18 +0100
Subject: [PATCH 04/67] Second attempt at having the bot run the test suite,
 now based on Thomas' PR 366

---
 bot/check-test.sh | 66 ++++++++++++++++++++++++++++++++++
 run_tests.sh      | 10 +++++-
 test_suite.sh     | 92 +++++++++++++++++++++++++++++------------------
 3 files changed, 132 insertions(+), 36 deletions(-)
 create mode 100644 bot/check-test.sh

diff --git a/bot/check-test.sh b/bot/check-test.sh
new file mode 100644
index 0000000000..9fc783aa99
--- /dev/null
+++ b/bot/check-test.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Dummy script that only creates test result file for the bot, without actually checking anything
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Kenneth Hoste (HPC-UGent)
+#
+# license: GPLv2
+#
+job_dir=${PWD}
+job_out="slurm-${SLURM_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
+
+# ReFrame prints e.g.
+#[----------] start processing checks
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
+#[     FAIL ] (1/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
+#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_d597cff4'
+#[       OK ] (2/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
+#P: perf: 8.441 ns/day (r:0, l:None, u:None)
+#[     FAIL ] (3/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
+#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_f4194106'
+#[----------] all spawned checks have finished
+#[  FAILED  ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted)
+
+# We will grep for the last and final line, since this reflects the overall result
+# Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
+FAILED=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_failed='\[\s*FAILED\s*\]'
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+
+# Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
+# I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
+ERROR=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_error='ERROR: '
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
+  [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+echo "[TEST]" > ${job_test_result_file}
+if [[ ${ERROR} -eq 1 ]]; then
+    echo "comment_description = Failure to execute test step" >> ${job_test_result_file}
+    echo "status = FAILURE" >> ${job_test_result_file}
+elif [[ ${FAILED} -eq 1 ]]; then
+    echo "comment_description = EESSI test suite produced failures" >> ${job_test_result_file}
+else
+    echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
+    echo "status = SUCCESS" >> ${job_test_result_file}
+fi
+
+exit 0
diff --git a/run_tests.sh b/run_tests.sh
index ccbc751f22..69672f18f3 100644
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,4 +1,12 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
-./run_in_compat_layer_env.sh ./test_suite.sh "$@"
+
+# Note: for these tests, we _don't_ run in the compat layer env
+# These tests should mimic what users do, and they are typically not in a prefix environment
+
+# Run eb --sanity-check-only on changed easyconfigs
+# TODO: in the future we may implement this as a light first check.
+
+# Run the test suite
+./test_suite.sh "$@"
diff --git a/test_suite.sh b/test_suite.sh
index fcaaa339d2..36262a6cbf 100644
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -144,41 +144,63 @@ else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
-# assume there's only one diff file that corresponds to the PR patch file
-pr_diff=$(ls [0-9]*.diff | head -1)
-
-# "split" the file by prefixing each line belonging to the same file with the
-# same number
-split_file=$(awk '/^\+\+\+/{n++}{print n, "  ", $0 }' ${pr_diff})
-
-# determine which easystack files may have changed
-changed_es_files=$(echo "${split_file}" | grep '^[0-9 ]*+++ ./eessi.*.yml$' | egrep -v 'known-issues|missing')
-
-# process all changed easystackfiles
-for es_file_num in $(echo "${changed_es_files}" | cut -f1 -d' ')
-do
-    # determine added lines that do not contain a yaml comment only
-    added_lines=$(echo "${split_file}" | grep "${es_file_num}    + " | sed -e "s/^"${es_file_num}"    + //" | grep -v "^[ ]*#")
-    # determine easyconfigs
-    easyconfigs=$(echo "${added_lines}" | cut -f3 -d' ')
-    # get easystack file name
-    easystack_file=$(echo "${changed_es_files}" | grep "^${es_file_num}" | sed -e "s/^"${es_file_num}"    ... .\///")
-    echo -e "Processing easystack file ${easystack_file}...\n\n"
-
-    # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-    eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
-
-    # load EasyBuild module
-    module load EasyBuild/${eb_version}
-
-    echo_green "All set, let's run sanity checks for installed packages..."
-
-    for easyconfig in ${easyconfigs};
-    do
-        echo "Running sanity check for '${easyconfig}'..."
-        eb --sanity-check-only ${easyconfig}
-    done
-done
+# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version
+# of the ReFrame module available in the current environment
+module load ReFrame/4.3.3
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Loaded ReFrame/4.3.3"
+else
+    fatal_error "Failed to load the ReFrame module"
+fi
+
+# Check ReFrame came with the hpctestlib and we can import it
+python3 -c 'import hpctestlib.sciapps.gromacs'
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully found and imported hpctestlib.sciapps.gromas"
+else
+    fatal_error "Failed to load hpctestlib"
+fi
+
+# Clone the EESSI test suite
+git clone https://github.com/EESSI/test-suite EESSI-test-suite
+export TESTSUITEPREFIX=$PWD/EESSI-test-suite
+export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
+
+# Check that we can import from the testsuite
+python3 -c 'import eessi.testsuite'
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully found and imported eessi.testsuite"
+else
+    fatal_error "FAILED to import from eessi.testsuite in Python"
+fi
+
+# Configure ReFrame
+export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py
+export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
+export RFM_CHECK_SEARCH_RECURSIVE=1
+export RFM_PREFIX=$PWD/reframe_runs
+
+# Check we can run reframe
+reframe --version
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully ran reframe --version"
+else
+    fatal_error "Failed to run ReFrame --version"
+fi
+
+# List the tests we want to run
+export REFRAME_ARGS='--tag CI --tag 1_nodes'
+reframe "${REFRAME_ARGS}" --list
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
+else
+    fatal_error "Failed to list ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
+fi
+
+# Run all tests
+reframe "${REFRAME_ARGS}" --run
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
+
+exit 0

From 34fca15420fb37eeb136da55542b44b0a474a63d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 5 Feb 2024 17:35:29 +0100
Subject: [PATCH 05/67] Make a very simple change to an easystack file so we
 are able to trigger the bot build and test procedures and see if this PR
 actually works

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 82190071ab..4c73b5887a 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -15,3 +15,4 @@ easyconfigs:
       options:
         from-pr: 19573
   - scikit-learn-1.3.1-gfbf-2023a.eb
+  - patchelf-0.18.0-GCCcore-12.3.0.eb

From 3faae9de814b96cd07896bd4042ae7b5b4b50f63 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:00:25 +0100
Subject: [PATCH 06/67] Make scripts executable

---
 bot/check-test.sh | 0
 run_tests.sh      | 0
 test_suite.sh     | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 bot/check-test.sh
 mode change 100644 => 100755 run_tests.sh
 mode change 100644 => 100755 test_suite.sh

diff --git a/bot/check-test.sh b/bot/check-test.sh
old mode 100644
new mode 100755
diff --git a/run_tests.sh b/run_tests.sh
old mode 100644
new mode 100755
diff --git a/test_suite.sh b/test_suite.sh
old mode 100644
new mode 100755

From 081e2b195b1dd948ae86d4f8e1634d368db3b467 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:30:25 +0100
Subject: [PATCH 07/67] Check for existence of SLURM output first

---
 bot/check-test.sh | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9fc783aa99..9308a87e8e 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -13,6 +13,17 @@ job_dir=${PWD}
 job_out="slurm-${SLURM_JOB_ID}.out"
 job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
 
+# Check that job output file is found
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
+if  [[ -f ${job_out} ]]; then
+    SLURM=1
+    [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
+else
+    SLURM=0
+    [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
+fi
+
+
 # ReFrame prints e.g.
 #[----------] start processing checks
 #[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
@@ -39,7 +50,6 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
-
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
 # I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
 ERROR=-1
@@ -53,11 +63,13 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 echo "[TEST]" > ${job_test_result_file}
-if [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = Failure to execute test step" >> ${job_test_result_file}
+if [[ ${SLURM} -eq 0 ]]; then
+    echo "comment_description = FAILED (job output file not found)" >> ${job_test_result_file}
+elif [[ ${ERROR} -eq 1 ]]; then
+    echo "comment_description = FAILED (test step failed to execute)" >> ${job_test_result_file}
     echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = EESSI test suite produced failures" >> ${job_test_result_file}
+    echo "comment_description = FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
 else
     echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
     echo "status = SUCCESS" >> ${job_test_result_file}

From 9b6fa731d6ac55e3e57538eda01c9c47d73300b1 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:43:20 +0100
Subject: [PATCH 08/67] Clarify return messages from the bot

---
 bot/check-test.sh |  8 ++++----
 test_suite.sh     | 20 --------------------
 2 files changed, 4 insertions(+), 24 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9308a87e8e..05c7589640 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -64,14 +64,14 @@ fi
 
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
-    echo "comment_description = FAILED (job output file not found)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (job output file not found, cannot check test results)" >> ${job_test_result_file}
 elif [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = FAILED (test step failed to execute)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (EESSI test suite was not run, test step itself failed to execute)" >> ${job_test_result_file}
     echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
 else
-    echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
+    echo "comment_description = :grin: SUCCESS" >> ${job_test_result_file}
     echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
diff --git a/test_suite.sh b/test_suite.sh
index 36262a6cbf..7995f8aa3a 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -78,26 +78,9 @@ else
     fatal_error "$EESSI_CVMFS_REPO is not available!"
 fi
 
-# make sure we're in Prefix environment by checking $SHELL
-if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
-    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
-else
-    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
-fi
-
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
 export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 
-DETECTION_PARAMETERS=''
-GENERIC=0
-EB='eb'
-if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
-    echo_yellow ">> GENERIC build/test requested, taking appropriate measures!"
-    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
-    GENERIC=1
-    EB='eb --optarch=GENERIC'
-fi
-
 echo ">> Determining software subdirectory to use for current build/test host..."
 if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
   export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
@@ -129,9 +112,6 @@ else
     fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
 fi
 
-echo ">> Configuring EasyBuild..."
-source $TOPDIR/configure_easybuild
-
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge

From cb48b36e590f1d36d7a8d08df22eb538b716c70b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 11:32:15 +0100
Subject: [PATCH 09/67] Use EESSI_SOFTWARE_PATH instead of
 EASYBUILD_INSTALLPATH to set the modulepath. It should be the same, and we no
 longer configure EasyBuild, since we don't use it when running the test suite

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 7995f8aa3a..1464a3a5d4 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -117,7 +117,7 @@ echo ">> Setting up \$MODULEPATH..."
 module --force purge
 # ignore current $MODULEPATH entirely
 module unuse $MODULEPATH
-module use $EASYBUILD_INSTALLPATH/modules/all
+module use ${EESSI_SOFTWARE_PATH}/modules/all
 if [[ -z ${MODULEPATH} ]]; then
     fatal_error "Failed to set up \$MODULEPATH?!"
 else

From d1a0219486fe4a4c1ad94039b4b74b41ebf7da1f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:02:12 +0100
Subject: [PATCH 10/67] Do git clone in a seperate script, so that that can be
 run in the prefix layer

---
 clone_eessi_test_suite.sh | 1 +
 run_tests.sh              | 3 ++-
 test_suite.sh             | 9 +++++++--
 3 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100755 clone_eessi_test_suite.sh

diff --git a/clone_eessi_test_suite.sh b/clone_eessi_test_suite.sh
new file mode 100755
index 0000000000..f7684c8cfd
--- /dev/null
+++ b/clone_eessi_test_suite.sh
@@ -0,0 +1 @@
+git clone https://github.com/EESSI/test-suite EESSI-test-suite
diff --git a/run_tests.sh b/run_tests.sh
index 69672f18f3..90185fea17 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,4 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./test_suite.sh "$@"
+./run_in_compat_layer_env.sh clone_eessi_test_suite.sh
+./test_suite.sh
diff --git a/test_suite.sh b/test_suite.sh
index 1464a3a5d4..6177f0a253 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -141,9 +141,14 @@ else
     fatal_error "Failed to load hpctestlib"
 fi
 
-# Clone the EESSI test suite
-git clone https://github.com/EESSI/test-suite EESSI-test-suite
+# Cloning should already be done by clone_eessi_test_suite.sh, which runs in compat layer to have 'git' available
+# git clone https://github.com/EESSI/test-suite EESSI-test-suite
 export TESTSUITEPREFIX=$PWD/EESSI-test-suite
+if [ -d $TESTSUITEPREFIX ]; then
+    echo_green "Clone of the test suite $TESTSUITEPREFIX available, OK!"
+else
+    fatal_error "Clone of the test suite $TESTSUITEPREFIX is not available!"
+fi
 export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
 
 # Check that we can import from the testsuite

From 86d5d3d47d076fbdf0a61c57ca83fee8a1c87123 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:06:09 +0100
Subject: [PATCH 11/67] Should use the one from current dir

---
 run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run_tests.sh b/run_tests.sh
index 90185fea17..bcca1ee417 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,5 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./run_in_compat_layer_env.sh clone_eessi_test_suite.sh
+./run_in_compat_layer_env.sh ./clone_eessi_test_suite.sh
 ./test_suite.sh

From 7861aec18c0081c8ff24e7ed46d34f9ea52c2dae Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:10:56 +0100
Subject: [PATCH 12/67] Remove quotes, see if that helps

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 6177f0a253..1ac82a5e66 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe "${REFRAME_ARGS}" --list
+reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else

From a7b5ee13c841e940ab136ee2d014b4426a23baa7 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:52:41 +0100
Subject: [PATCH 13/67] Apparently, we need to remove curly braces too

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1ac82a5e66..46c3493f41 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe ${REFRAME_ARGS} --list
+reframe $REFRAME_ARGS --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else

From f50e463e9d0d0f5d91b161805279b6a8a764b990 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:03:31 +0100
Subject: [PATCH 14/67] Make sure this actually gets reported as failure, as
 the ReFrame runtime fails to run the test suite (it is not an indication of
 tests failing themselves).

---
 test_suite.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 46c3493f41..8a69bdee8d 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe $REFRAME_ARGS --list
+reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else
@@ -184,6 +184,12 @@ fi
 
 # Run all tests
 reframe "${REFRAME_ARGS}" --run
+if [[ $? -eq 0 ]]; then
+    echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
+else
+    fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
+fi
+
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}

From f67df9bc1319053901aa506668533cbb8b99716f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:04:49 +0100
Subject: [PATCH 15/67] Fix the actual issue: make sure that two arguments are
 interpreted seperately by not quoting them

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 8a69bdee8d..633333890c 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -183,7 +183,7 @@ else
 fi
 
 # Run all tests
-reframe "${REFRAME_ARGS}" --run
+reframe ${REFRAME_ARGS} --run
 if [[ $? -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
 else

From 3394851c5347a045431d7826f579ad5f35f01edf Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:32:27 +0100
Subject: [PATCH 16/67] See if we can make check-test formatting more fancy

---
 bot/check-test.sh | 84 +++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 78 insertions(+), 6 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 05c7589640..3fa7c1d694 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -64,15 +64,87 @@ fi
 
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
-    echo "comment_description = :cry: FAILED (job output file not found, cannot check test results)" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(job output file not found, cannot check test results)"# >> ${job_test_result_file}
+    status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = :cry: FAILED (EESSI test suite was not run, test step itself failed to execute)" >> ${job_test_result_file}
-    echo "status = FAILURE" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(EESSI test suite was not run, test step itself failed to execute)"# >> ${job_test_result_file}
+    status="FAILURE"
+#    echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = :cry: FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(EESSI test suite produced failures)"# >> ${job_test_result_file}
+    status="FAILURE"
 else
-    echo "comment_description = :grin: SUCCESS" >> ${job_test_result_file}
-    echo "status = SUCCESS" >> ${job_test_result_file}
+    summary=":grin: SUCCESS"# >> ${job_test_result_file}
+    summary_details=""
+    status="SUCCESS"
+#    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+function add_detail() {
+    actual=${1}
+    expected=${2}
+    success_msg="${3}"
+    failure_msg="${4}"
+    if [[ ${actual} -eq ${expected} ]]; then
+        success "${success_msg}"
+    else
+        failure "${failure_msg}"
+    fi
+}
+
+echo "[TEST]" > ${job_result_file}
+echo -n "comment_description = " >> ${job_result_file}
+
+# Use template for writing PR comment with details
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_success_item_fmt=":white_check_mark: __ITEM__"
+comment_failure_item_fmt=":x: __ITEM__"
+comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
+comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# first construct comment_details_list, abbreviated CoDeList
+# then use it to set comment_details
+CoDeList=""
+
+success_msg="job output file <code>${job_out}</code>"
+failure_msg="no job output file <code>${job_out}</code>"
+CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_error}</code>"
+failure_msg="found message matching <code>${GP_error}</code>"
+CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_failed}</code>"
+failure_msg="found message matching <code>${GP_failed}</code>"
+CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+
+# Should not be needed for testing, I think? Maybe for loading ReFrame module...
+# success_msg="no message matching <code>${GP_req_missing}</code>"
+# failure_msg="found message matching <code>${GP_req_missing}</code>"
+# CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+# 
+# success_msg="found message(s) matching <code>${GP_no_missing}</code>"
+# failure_msg="no message matching <code>${GP_no_missing}</code>"
+# CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+# 
+# success_msg="found message matching <code>${GP_tgz_created}</code>"
+# failure_msg="no message matching <code>${GP_tgz_created}</code>"
+# CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+
+comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
+comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
+
+# Actually writing the comment description to the result file
+echo "${comment_description}" >> ${job_result_file}
+
 exit 0

From 8e5682a47a589a2e251852ed674b71f16980ffc7 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 14:51:46 +0100
Subject: [PATCH 17/67] Replace the env var for the result file

---
 bot/check-test.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 3fa7c1d694..89d3d6a3c6 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -95,8 +95,8 @@ function add_detail() {
     fi
 }
 
-echo "[TEST]" > ${job_result_file}
-echo -n "comment_description = " >> ${job_result_file}
+echo "[TEST]" > ${job_test_result_file}
+echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
@@ -145,6 +145,6 @@ comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file
-echo "${comment_description}" >> ${job_result_file}
+echo "${comment_description}" >> ${job_test_result_file}
 
 exit 0

From 6cf0cf959844081b004595bd3a6c21329ccf256d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 14:53:21 +0100
Subject: [PATCH 18/67] Add missing functions

---
 bot/check-test.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 89d3d6a3c6..87c79b3b30 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -83,6 +83,24 @@ else
 #    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+function print_br_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "${format//__ITEM__/${item}}<br/>"
+}
+
+function success() {
+    format="${comment_success_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
+function failure() {
+    format="${comment_failure_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
 function add_detail() {
     actual=${1}
     expected=${2}

From 7fe24e13cdef3f703b51930423a2296e3dc07d59 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:14:50 +0100
Subject: [PATCH 19/67] Remove artefacts from reporting

---
 bot/check-test.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 87c79b3b30..da06cd630b 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -118,13 +118,11 @@ echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
 comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_success_item_fmt=":white_check_mark: __ITEM__"
 comment_failure_item_fmt=":x: __ITEM__"
-comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
-comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
 
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 

From afc309a168048e54b625ec857a5d09a14ca1d585 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:25:33 +0100
Subject: [PATCH 20/67] Add a reason for the failure

---
 bot/check-test.sh | 42 ++++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index da06cd630b..38e86efccd 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -65,24 +65,37 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(job output file not found, cannot check test results)"# >> ${job_test_result_file}
+    summary_details="Reason: job output file not found, cannot check test results."
     status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(EESSI test suite was not run, test step itself failed to execute)"# >> ${job_test_result_file}
+    summary_details="Reason: EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
-#    echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(EESSI test suite produced failures)"# >> ${job_test_result_file}
+    summary_details="Reason: EESSI test suite produced failures."
     status="FAILURE"
 else
-    summary=":grin: SUCCESS"# >> ${job_test_result_file}
+    summary=":grin: SUCCESS"
     summary_details=""
     status="SUCCESS"
-#    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+
+echo "[TEST]" > ${job_test_result_file}
+echo -n "comment_description = " >> ${job_test_result_file}
+
+# Use template for writing PR comment with details
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_success_item_fmt=":white_check_mark: __ITEM__"
+comment_failure_item_fmt=":x: __ITEM__"
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# Declare functions
 function print_br_item() {
     format="${1}"
     item="${2}"
@@ -113,22 +126,11 @@ function add_detail() {
     fi
 }
 
-echo "[TEST]" > ${job_test_result_file}
-echo -n "comment_description = " >> ${job_test_result_file}
-
-# Use template for writing PR comment with details
-# construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
-comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
-comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
-comment_success_item_fmt=":white_check_mark: __ITEM__"
-comment_failure_item_fmt=":x: __ITEM__"
-
-comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
-
 # first construct comment_details_list, abbreviated CoDeList
 # then use it to set comment_details
-CoDeList=""
+
+# Initialize with summary_details, which elaborates on the reason for failure
+CoDeList=$(print_br_item "__ITEM__" "${summary_details}"
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From b84e48755dfb582a5a4bae3480e8865df179331e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:32:07 +0100
Subject: [PATCH 21/67] Fixed missing bracket

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 38e86efccd..e340907bb3 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -130,7 +130,7 @@ function add_detail() {
 # then use it to set comment_details
 
 # Initialize with summary_details, which elaborates on the reason for failure
-CoDeList=$(print_br_item "__ITEM__" "${summary_details}"
+CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From ef7bc0142f0ff518e82121371403c4da9bc0444d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:48:44 +0100
Subject: [PATCH 22/67] Add reporting of the ReFrame result

---
 bot/check-test.sh | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index e340907bb3..78c153e5dc 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -43,11 +43,11 @@ fi
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_failed='\[\s*FAILED\s*\]'
-  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
 fi
 
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
@@ -62,6 +62,15 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
+# Grep for the success pattern, so we can report the amount of tests run
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_success='\[\s*PASSED\s*\]'
+  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+fi
+
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
@@ -132,6 +141,11 @@ function add_detail() {
 # Initialize with summary_details, which elaborates on the reason for failure
 CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
+# Add final ReFrame line as line
+if [[ ! -z ${grep_reframe_result} ]]; then
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}"
+fi
+
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
 CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
@@ -140,8 +154,8 @@ success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
 CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
-success_msg="no message matching <code>${GP_failed}</code>"
-failure_msg="found message matching <code>${GP_failed}</code>"
+success_msg="no message matching <code>""${GP_failed}""</code>"
+failure_msg="found message matching <code>""${GP_failed}""</code>"
 CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 # Should not be needed for testing, I think? Maybe for loading ReFrame module...

From 867681a2ed7413b8ecdbf646d05a20d7d2553adc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:01:40 +0100
Subject: [PATCH 23/67] Add missing bracket. Again

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 78c153e5dc..7ff5f8a3ea 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -143,7 +143,7 @@ CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
 # Add final ReFrame line as line
 if [[ ! -z ${grep_reframe_result} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}"
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
 fi
 
 success_msg="job output file <code>${job_out}</code>"

From 47e30cac825362f85138e1d6cb3fe8ba02f13c6e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:17:25 +0100
Subject: [PATCH 24/67] Add some echo for debugging

---
 bot/check-test.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7ff5f8a3ea..1a2d8b5d5d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -171,6 +171,9 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
 # CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
+echo "Finale CoDeList is:"
+echo ${CoDeList}
+
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
 
 comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}

From 1bdadf884e8f18ec911f8961a894125a6c5cccce Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:24:34 +0100
Subject: [PATCH 25/67] more debugging output

---
 bot/check-test.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 1a2d8b5d5d..841b8ed01b 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -137,14 +137,20 @@ function add_detail() {
 
 # first construct comment_details_list, abbreviated CoDeList
 # then use it to set comment_details
+CoDeList=""
 
 # Initialize with summary_details, which elaborates on the reason for failure
-CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
+if [[ ! -z ${summary_details} ]]; then
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
+fi
 
 # Add final ReFrame line as line
 if [[ ! -z ${grep_reframe_result} ]]; then
     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
 fi
+echo "CoDeList up here is"
+echo ${CoDeList}
+
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From 5152f5e47380c462c234566f5fc3c29922694ef1 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:54:02 +0100
Subject: [PATCH 26/67] Lets try some nicer formatting with headers etc. See if
 this actually prints something.

---
 bot/check-test.sh | 62 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 20 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 841b8ed01b..8f8eb66f5e 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -74,19 +74,19 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: job output file not found, cannot check test results."
+    reason="Reason: job output file not found, cannot check test results."
     status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: EESSI test suite was not run, test step itself failed to execute."
+    reason="Reason: EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
 elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: EESSI test suite produced failures."
+    reason="Reason: EESSI test suite produced failures."
     status="FAILURE"
 else
     summary=":grin: SUCCESS"
-    summary_details=""
+    reason=""
     status="SUCCESS"
 fi
 
@@ -96,13 +96,36 @@ echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
-comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
-comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_template="<details>__SUMMARY_FMT__<dl>__REASON_FMT____REFRAME_FMT____DETAILS_FMT__</dl></details>"
 comment_success_item_fmt=":white_check_mark: __ITEM__"
 comment_failure_item_fmt=":x: __ITEM__"
 
+# Initialize comment_description
+comment_description=${comment_template}
+
+# Now, start replacing template items one by one
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}}
+
+
+# Omit this if there is no reason (e.g. because it was succesful)
+if [[ -z ${reason} ]]; then
+    comment_reason_fmt="<dt>_Reason_</dt><dd>__REASONS__</dd>"
+    reason_details="${comment_reason_fmt/__REASONS__/${reason}}"
+    comment_description=${comment_description/__REASON_FMT__/${reason_details}}
+else
+    comment_description=${comment_description/__REASON_FMT__/""}
+fi
+
+# Omit this if there is no reframe summary (i.e. the workflow didn't run succesfully)
+if [[ -z ${grep_reframe_result} ]]; then
+    comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
+    reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
+    comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
+else
+    comment_description=${comment_description/__REFRAME_FMT__/""}
+fi
 
 # Declare functions
 function print_br_item() {
@@ -139,17 +162,17 @@ function add_detail() {
 # then use it to set comment_details
 CoDeList=""
 
-# Initialize with summary_details, which elaborates on the reason for failure
-if [[ ! -z ${summary_details} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
-fi
-
-# Add final ReFrame line as line
-if [[ ! -z ${grep_reframe_result} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
-fi
-echo "CoDeList up here is"
-echo ${CoDeList}
+# # Initialize with summary_details, which elaborates on the reason for failure
+# if [[ ! -z ${summary_details} ]]; then
+#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
+# fi
+# 
+# # Add final ReFrame line as line
+# if [[ ! -z ${grep_reframe_result} ]]; then
+#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
+# fi
+# echo "CoDeList up here is"
+# echo ${CoDeList}
 
 
 success_msg="job output file <code>${job_out}</code>"
@@ -180,9 +203,8 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 echo "Finale CoDeList is:"
 echo ${CoDeList}
 
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
-
-comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file

From ea38c66114af601071e1edc626cf9952219a7a18 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:07:28 +0100
Subject: [PATCH 27/67] Inverse check, it was incorrect

---
 bot/check-test.sh | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 8f8eb66f5e..01769bbac3 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -109,8 +109,8 @@ comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}}
 
 
-# Omit this if there is no reason (e.g. because it was succesful)
-if [[ -z ${reason} ]]; then
+# Only add if there is a reason (e.g. no reason for successful runs)
+if [[ ! -z ${reason} ]]; then
     comment_reason_fmt="<dt>_Reason_</dt><dd>__REASONS__</dd>"
     reason_details="${comment_reason_fmt/__REASONS__/${reason}}"
     comment_description=${comment_description/__REASON_FMT__/${reason_details}}
@@ -118,8 +118,8 @@ else
     comment_description=${comment_description/__REASON_FMT__/""}
 fi
 
-# Omit this if there is no reframe summary (i.e. the workflow didn't run succesfully)
-if [[ -z ${grep_reframe_result} ]]; then
+# Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully)
+if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
     reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
     comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
@@ -162,19 +162,6 @@ function add_detail() {
 # then use it to set comment_details
 CoDeList=""
 
-# # Initialize with summary_details, which elaborates on the reason for failure
-# if [[ ! -z ${summary_details} ]]; then
-#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
-# fi
-# 
-# # Add final ReFrame line as line
-# if [[ ! -z ${grep_reframe_result} ]]; then
-#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
-# fi
-# echo "CoDeList up here is"
-# echo ${CoDeList}
-
-
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
 CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")

From 8edb6fffb401f0e74e0b975b5585d3f9257f88f8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:11:41 +0100
Subject: [PATCH 28/67] Remove debugging output

---
 bot/check-test.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 01769bbac3..7a663f35d1 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -187,9 +187,6 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
 # CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
-echo "Finale CoDeList is:"
-echo ${CoDeList}
-
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}

From 26bd2cb04af381af580805b094f69c096ea40410 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:33:12 +0100
Subject: [PATCH 29/67] Correct the regex pattern to account for the fact that
 the slurm output file contains special characters in that pattern that are
 NOT whitespace

---
 bot/check-test.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7a663f35d1..aebd56d333 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -42,7 +42,7 @@ fi
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
-  GP_failed='\[\s*FAILED\s*\]'
+  GP_failed='\[.*FAILED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
@@ -64,7 +64,7 @@ fi
 
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
-  GP_success='\[\s*PASSED\s*\]'
+  GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
@@ -119,6 +119,8 @@ else
 fi
 
 # Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully)
+echo "ReFrame result:"
+echo "${grep_reframe_result}"
 if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
     reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}

From 416b72d3cb18d6150e0ef8760cd5b66867e29496 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:45:30 +0100
Subject: [PATCH 30/67] Corrected typo

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index aebd56d333..702add848d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -123,7 +123,7 @@ echo "ReFrame result:"
 echo "${grep_reframe_result}"
 if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
-    reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
+    reframe_summary=${comment_reframe_fmt/__REFRAME_SUMMARY__/${grep_reframe_result}}
     comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
 else
     comment_description=${comment_description/__REFRAME_FMT__/""}

From 19900937361e16259dce714c9e4acda4b09e0a7b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:51:27 +0100
Subject: [PATCH 31/67] Trigger failure in the workflow on purpose, to show
 what that looks like

---
 test_suite.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 633333890c..4439821d56 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -166,7 +166,7 @@ export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
 
 # Check we can run reframe
-reframe --version
+reframe --version BLABLABLA
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully ran reframe --version"
 else
@@ -174,7 +174,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_nodes'
+export REFRAME_ARGS='--tag CI --tag 1_node'
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"

From 4b9885612fa13d2d1f45522ca1aa04ec333863b8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 18:11:30 +0100
Subject: [PATCH 32/67] Change order of checking error codes. If the ReFrame
 tests ran and we find that pattern, that should take priority. Other errors
 might have come from errors in the build phase...

---
 bot/check-test.sh | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 702add848d..7b7ada98e7 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -62,10 +62,12 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
+SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
@@ -74,19 +76,27 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: job output file not found, cannot check test results."
+    reason="Job output file not found, cannot check test results."
     status="FAILURE"
-elif [[ ${ERROR} -eq 1 ]]; then
+# Should come before general errors: if SUCCESS==1, it indicates the test suite ran succesfully
+# regardless of other things that might have gone wrong
+elif [[ ${SUCCESS} -eq 1 ]]; then
+    summary=":grin: SUCCESS"
+    reason=""
+    status="SUCCESS"
+# Should come before general errors: if FAILED==1, it indicates the test suite ran
+# otherwise the pattern wouldn't have been there
+elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: EESSI test suite was not run, test step itself failed to execute."
+    reason="EESSI test suite produced failures."
     status="FAILURE"
-elif [[ ${FAILED} -eq 1 ]]; then
+elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: EESSI test suite produced failures."
+    reason="EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
 else
-    summary=":grin: SUCCESS"
-    reason=""
+    summary=":grin: FAILURE"
+    reason="Failed for unknown reason"
     status="SUCCESS"
 fi
 

From 0f261420106862049091494d6a640a6c46de5dd8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 12:11:25 +0100
Subject: [PATCH 33/67] Make sure we don't overwrite the grepped result from
 failures

---
 bot/check-test.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7b7ada98e7..8af45d26a6 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -43,11 +43,11 @@ fi
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_failed='\[.*FAILED.*\].*Ran .* test case'
-  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_failed}"
 fi
 
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
@@ -66,11 +66,17 @@ SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
-  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_success}"
+fi
+
+if [[ ! -z ${grep_reframe_failed} ]]; then
+    grep_reframe_results=${grep_reframe_failed}
+else
+    grep_reframe_results=${grep_reframe_success}
 fi
 
 echo "[TEST]" > ${job_test_result_file}

From 94e0bb5e3b886192508d95485b221b3ec7a6ff5e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 13:09:31 +0100
Subject: [PATCH 34/67] Fix plural/singular typo

---
 bot/check-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 8af45d26a6..39901f0691 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -74,9 +74,9 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 if [[ ! -z ${grep_reframe_failed} ]]; then
-    grep_reframe_results=${grep_reframe_failed}
+    grep_reframe_result=${grep_reframe_failed}
 else
-    grep_reframe_results=${grep_reframe_success}
+    grep_reframe_result=${grep_reframe_success}
 fi
 
 echo "[TEST]" > ${job_test_result_file}

From 9e7ac1c7c4f2f64250ef504810d057e0cb3e5225 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 13:25:45 +0100
Subject: [PATCH 35/67] Be a bit more expressive about what we are doing

---
 test_suite.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 4439821d56..1610d87738 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -165,8 +165,11 @@ export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
 
+echo "Configured reframe with the following environment variables:"
+env | grep "RFM_"
+
 # Check we can run reframe
-reframe --version BLABLABLA
+reframe --version
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully ran reframe --version"
 else
@@ -175,6 +178,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_node'
+echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
@@ -183,6 +187,7 @@ else
 fi
 
 # Run all tests
+echo "Running tests: reframe ${REFRAME_ARGS} --run"
 reframe ${REFRAME_ARGS} --run
 if [[ $? -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
@@ -190,7 +195,6 @@ else
     fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
 fi
 
-
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
 

From 3a52bde23e5de6cfe4d5da170060c07d8dfc0009 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 14:28:08 +0100
Subject: [PATCH 36/67] Change to dedicated config file for testing by the bot
 in the build job

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1610d87738..d6699878f9 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -160,7 +160,7 @@ else
 fi
 
 # Configure ReFrame
-export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py
+export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/software_layer_bot.py
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs

From 159e384bdef117265e09a0a420cce3fa6d8842c8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 14:47:54 +0100
Subject: [PATCH 37/67] Make it verbose, see if we can see why autodetect fails

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index d6699878f9..a8e1b2d97a 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -177,7 +177,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node'
+export REFRAME_ARGS='--tag CI --tag 1_node -vvv'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From b92044e5f913ce0a6c7082c0ebfb4e238d47f95d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:23:53 +0100
Subject: [PATCH 38/67] Add ReFrame config template. Not that four template
 variables need to be replaced for this to be a valid ReFrame config file

---
 reframe_config_bot.py | 58 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 reframe_config_bot.py

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
new file mode 100644
index 0000000000..65ccf9f43e
--- /dev/null
+++ b/reframe_config_bot.py
@@ -0,0 +1,58 @@
+# WARNING: this file is intended as template and the __X__ template variables need to be replaced
+# before it can act as a configuration file
+# Once replaced, this is a config file for running tests after the build phase, by the bot
+
+from eessi.testsuite.common_config import common_logging_config
+from eessi.testsuite.constants import *  # noqa: F403
+
+
+site_configuration = {
+    'systems': [
+        {
+            'name': 'Testing in bot Build jobs for EESSI software Layer',
+            'descr': 'Software-layer bot',
+            'hostnames': ['.*'],
+            'modules_system': 'lmod',
+            'partitions': [
+                {
+                    'name': 'default',
+                    'scheduler': 'local',
+                    'launcher': 'mpirun',
+                    'environs': ['default'],
+                    'features': [FEATURES[CPU]],
+                    'processor': {
+                        'num_cpus': __NUM_CPUS__,
+                        'num_sockets': __NUM_SOCKETS__,
+                        'num_cpus_per_core': __NUM_CPUS_PER_CORE__,
+                        'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__,
+                    }
+                    'resources': [
+                        {
+                            'name': 'memory',
+                            'options': ['--mem={size}'],
+                        }
+                    ],
+                    'max_jobs': 1
+                    }
+                ]
+            }
+        ],
+    'environments': [
+        {
+            'name': 'default',
+            'cc': 'cc',
+            'cxx': '',
+            'ftn': ''
+            }
+        ],
+    'general': [
+        {
+            'purge_environment': True,
+            'resolve_module_conflicts': False,  # avoid loading the module before submitting the job
+            # Enable automatic detection of CPU architecture
+            # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information
+            'remote_detect': True,
+        }
+    ],
+    'logging': common_logging_config(),
+}

From 02b0e3115dfc81387554f0cb9fe2c292e60098ab Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:30:18 +0100
Subject: [PATCH 39/67] Use template config file and make replacements based on
 lscpu

---
 test_suite.sh | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index a8e1b2d97a..f5cc7fabfd 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -156,11 +156,11 @@ python3 -c 'import eessi.testsuite'
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully found and imported eessi.testsuite"
 else
-    fatal_error "FAILED to import from eessi.testsuite in Python"
+    fatal_error "Failed to import from eessi.testsuite in Python"
 fi
 
 # Configure ReFrame
-export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/software_layer_bot.py
+export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
@@ -168,6 +168,33 @@ export RFM_PREFIX=$PWD/reframe_runs
 echo "Configured reframe with the following environment variables:"
 env | grep "RFM_"
 
+# Inject correct CPU properties into the ReFrame config file
+cpuinfo=$(lscpu)
+if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then
+    cpu_count=${BASH_REMATCH[1]}
+else
+    fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
+    socket_count=${BASH_REMATCH[1]}
+else
+    fatal_error "Failed to get the number of sockets for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
+    threads_per_core=${BASH_REMATCH[2]}
+else
+    fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
+    cores_per_socket=${BASH_REMATCH[2]}
+else
+    fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
+fi
+sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKETS__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+
 # Check we can run reframe
 reframe --version
 if [[ $? -eq 0 ]]; then

From 58c4cfa772454405c456503c29a6fc8d59956910 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:39:46 +0100
Subject: [PATCH 40/67] Fix incorrect variable name

---
 test_suite.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index f5cc7fabfd..7ba5282327 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,17 +175,17 @@ if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then
 else
     fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
+if [[ "${cpuinfo}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
     socket_count=${BASH_REMATCH[1]}
 else
     fatal_error "Failed to get the number of sockets for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
+if [[ "${cpuinfo}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
     threads_per_core=${BASH_REMATCH[2]}
 else
     fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
+if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
     cores_per_socket=${BASH_REMATCH[2]}
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."

From 2fc121e86399b7c7c3cab6092b609a91a88db43f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:57:48 +0100
Subject: [PATCH 41/67] Missing comma

---
 reframe_config_bot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
index 65ccf9f43e..bbe8a7e2df 100644
--- a/reframe_config_bot.py
+++ b/reframe_config_bot.py
@@ -25,7 +25,7 @@
                         'num_sockets': __NUM_SOCKETS__,
                         'num_cpus_per_core': __NUM_CPUS_PER_CORE__,
                         'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__,
-                    }
+                    },
                     'resources': [
                         {
                             'name': 'memory',

From db08f4f39a618450c70f1d52f43c46f244011545 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:30:43 +0100
Subject: [PATCH 42/67] Corrected typo

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 7ba5282327..9bd95c6749 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -193,7 +193,7 @@ fi
 sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_SOCKETS__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version

From 6b3a6ccc985314e5c66c8097e8eb26e1c1fb3546 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:47:43 +0100
Subject: [PATCH 43/67] Remove verbose flags

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 9bd95c6749..99643c42b8 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -204,7 +204,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node -vvv'
+export REFRAME_ARGS='--tag CI --tag 1_node'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From a70b4f132e36f078cb3b4f0d802f96a877699050 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:55:16 +0100
Subject: [PATCH 44/67] Pick name without spaces, to avoid issues with
 directories

---
 reframe_config_bot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
index bbe8a7e2df..ad66a72041 100644
--- a/reframe_config_bot.py
+++ b/reframe_config_bot.py
@@ -9,7 +9,7 @@
 site_configuration = {
     'systems': [
         {
-            'name': 'Testing in bot Build jobs for EESSI software Layer',
+            'name': 'BotBuildTests',
             'descr': 'Software-layer bot',
             'hostnames': ['.*'],
             'modules_system': 'lmod',

From ce24e6d6de196ca191371b186b449b7a9d4aa5c9 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:50:39 +0100
Subject: [PATCH 45/67] Pick more meaningful name for SLURM variable, namely
 SLURM_OUTPUT_FOUND

---
 bot/check-build.sh | 22 +++++++++++-----------
 bot/check-test.sh  | 18 +++++++++---------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index ec1ca56bba..42dca46a08 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -95,18 +95,18 @@ job_dir=${PWD}
 
 [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
 
-job_out="slurm-${SLURM_JOB_ID}.out"
+job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
-    SLURM=1
+    SLURM_OUTPUT_FOUND=1
     [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
-    SLURM=0
+    SLURM_OUTPUT_FOUND=0
     [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
 ERROR=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_error='ERROR: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
   [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
@@ -116,7 +116,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 FAILED=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_failed='FAILED: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
@@ -126,7 +126,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 MISSING=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_req_missing=' required modules missing:'
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}")
   [[ $? -eq 0 ]] && MISSING=1 || MISSING=0
@@ -136,7 +136,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 NO_MISSING=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_no_missing='No missing installations'
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}")
   [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
@@ -147,7 +147,7 @@ fi
 
 TGZ=-1
 TARBALL=
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_tgz_created="\.tar\.gz created!"
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
   if [[ $? -eq 0 ]]; then
@@ -169,9 +169,9 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 
-job_result_file=_bot_job${SLURM_JOB_ID}.result
+job_result_file=_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.result
 
-if [[ ${SLURM} -eq 1 ]] && \
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${ERROR} -eq 0 ]] && \
    [[ ${FAILED} -eq 0 ]] && \
    [[ ${MISSING} -eq 0 ]] && \
@@ -340,7 +340,7 @@ CoDeList=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 39901f0691..549afd2f65 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -10,16 +10,16 @@
 # license: GPLv2
 #
 job_dir=${PWD}
-job_out="slurm-${SLURM_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
+job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
-    SLURM=1
+    SLURM_OUTPUT_FOUND=1
     [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
-    SLURM=0
+    SLURM_OUTPUT_FOUND=0
     [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
@@ -41,7 +41,7 @@ fi
 # We will grep for the last and final line, since this reflects the overall result
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_failed='\[.*FAILED.*\].*Ran .* test case'
   grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
@@ -53,7 +53,7 @@ fi
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
 # I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
 ERROR=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_error='ERROR: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
   [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
@@ -64,7 +64,7 @@ fi
 
 SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
@@ -80,7 +80,7 @@ else
 fi
 
 echo "[TEST]" > ${job_test_result_file}
-if [[ ${SLURM} -eq 0 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 0 ]]; then
     summary=":cry: FAILURE"
     reason="Job output file not found, cannot check test results."
     status="FAILURE"
@@ -182,7 +182,7 @@ CoDeList=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"

From 9881c2ad5f93c617c075e8df81d770033ec2a121 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:51:36 +0100
Subject: [PATCH 46/67] Correct erroneous status

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 549afd2f65..9724f45832 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -103,7 +103,7 @@ elif [[ ${ERROR} -eq 1 ]]; then
 else
     summary=":grin: FAILURE"
     reason="Failed for unknown reason"
-    status="SUCCESS"
+    status="FAILURE"
 fi
 
 

From 1446055cb5cd4306d61809c67b5dcda4ce930c8b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:53:09 +0100
Subject: [PATCH 47/67] Rename CoDeList variable to comment_details_list

---
 bot/check-build.sh | 18 +++++++++---------
 bot/check-test.sh  | 18 +++++++++---------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index 42dca46a08..584d4ff828 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -334,35 +334,35 @@ echo -n "comment_description = " >> ${job_result_file}
 
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 
-# first construct comment_details_list, abbreviated CoDeList
+# first construct comment_details_list, abbreviated comment_details_list
 # then use it to set comment_details
-CoDeList=""
+comment_details_list=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
-CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_failed}</code>"
 failure_msg="found message matching <code>${GP_failed}</code>"
-CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_req_missing}</code>"
 failure_msg="found message matching <code>${GP_req_missing}</code>"
-CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="found message(s) matching <code>${GP_no_missing}</code>"
 failure_msg="no message matching <code>${GP_no_missing}</code>"
-CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="found message matching <code>${GP_tgz_created}</code>"
 failure_msg="no message matching <code>${GP_tgz_created}</code>"
-CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
-comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 
 
 # first construct comment_artefacts_list, abbreviated CoArList
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9724f45832..fe1fcfe85d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -176,37 +176,37 @@ function add_detail() {
     fi
 }
 
-# first construct comment_details_list, abbreviated CoDeList
+# first construct comment_details_list, abbreviated comment_details_list
 # then use it to set comment_details
-CoDeList=""
+comment_details_list=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
-CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
-CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 # Should not be needed for testing, I think? Maybe for loading ReFrame module...
 # success_msg="no message matching <code>${GP_req_missing}</code>"
 # failure_msg="found message matching <code>${GP_req_missing}</code>"
-# CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
 # 
 # success_msg="found message(s) matching <code>${GP_no_missing}</code>"
 # failure_msg="no message matching <code>${GP_no_missing}</code>"
-# CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 # 
 # success_msg="found message matching <code>${GP_tgz_created}</code>"
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
-# CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
-comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file

From f80c0eb42f891c686cd1818ccbfc353e8bb29f05 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:54:00 +0100
Subject: [PATCH 48/67] Remove commented section, it is not relevant

---
 bot/check-test.sh | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index fe1fcfe85d..9e7a524607 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -192,19 +192,6 @@ success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
 comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
-# Should not be needed for testing, I think? Maybe for loading ReFrame module...
-# success_msg="no message matching <code>${GP_req_missing}</code>"
-# failure_msg="found message matching <code>${GP_req_missing}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
-# 
-# success_msg="found message(s) matching <code>${GP_no_missing}</code>"
-# failure_msg="no message matching <code>${GP_no_missing}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
-# 
-# success_msg="found message matching <code>${GP_tgz_created}</code>"
-# failure_msg="no message matching <code>${GP_tgz_created}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
-
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}

From ff5baace136c589becaf42adab32b4a0eed7afc6 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com>
Date: Thu, 8 Feb 2024 17:57:36 +0100
Subject: [PATCH 49/67] Update run_tests.sh

Avoid a one-line script (if possible)

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run_tests.sh b/run_tests.sh
index bcca1ee417..5e9fcb543d 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,5 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./run_in_compat_layer_env.sh ./clone_eessi_test_suite.sh
+./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 ./test_suite.sh

From 4adae1838edd1146d1a4a8c4322bffeb5e49db64 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:42:47 +0100
Subject: [PATCH 50/67] Fix mistake in replacing SLURM with SLURM_OUTPUT_FOUND

---
 bot/check-build.sh | 4 ++--
 bot/check-test.sh  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index 584d4ff828..1b46652049 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -95,7 +95,7 @@ job_dir=${PWD}
 
 [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
 
-job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
+job_out="slurm-${SLURM_JOB_ID}.out"
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
     SLURM_OUTPUT_FOUND=1
@@ -169,7 +169,7 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 
-job_result_file=_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.result
+job_result_file=_bot_job${SLURM_JOB_ID}.result
 
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${ERROR} -eq 0 ]] && \
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9e7a524607..faf6e440fa 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -10,8 +10,8 @@
 # license: GPLv2
 #
 job_dir=${PWD}
-job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.test"
+job_out="slurm-${SLURM_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM__JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"

From 9e6955f53e70d3848ca2199756150993c36212ec Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:52:44 +0100
Subject: [PATCH 51/67] Change name of the template for the ReFrame config file
 to make clear it is a template. Then, alter the sed commands so that they
 output to a new file: the actual config file

---
 reframe_config_bot.py => reframe_config_bot.py.tmpl | 0
 test_suite.sh                                       | 9 +++++----
 2 files changed, 5 insertions(+), 4 deletions(-)
 rename reframe_config_bot.py => reframe_config_bot.py.tmpl (100%)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py.tmpl
similarity index 100%
rename from reframe_config_bot.py
rename to reframe_config_bot.py.tmpl
diff --git a/test_suite.sh b/test_suite.sh
index 99643c42b8..b3884c3b81 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -161,6 +161,7 @@ fi
 
 # Configure ReFrame
 export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
+export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
@@ -190,10 +191,10 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
 fi
-sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version

From 08a7fd8eaa64dce298624e1d4f6054d2938a7fcc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:55:20 +0100
Subject: [PATCH 52/67] Fix typo in variable

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index faf6e440fa..ef52206346 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -11,7 +11,7 @@
 #
 job_dir=${PWD}
 job_out="slurm-${SLURM_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM__JOB_ID}.test"
+job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"

From 8b5bde5413553c03f58ce788ef03e78ae87f343b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:56:07 +0100
Subject: [PATCH 53/67] No longer needed

---
 clone_eessi_test_suite.sh | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100755 clone_eessi_test_suite.sh

diff --git a/clone_eessi_test_suite.sh b/clone_eessi_test_suite.sh
deleted file mode 100755
index f7684c8cfd..0000000000
--- a/clone_eessi_test_suite.sh
+++ /dev/null
@@ -1 +0,0 @@
-git clone https://github.com/EESSI/test-suite EESSI-test-suite

From 5d383eb8bc4c729cfa85d1aeae9ea10d82000c5c Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:03:29 +0100
Subject: [PATCH 54/67] Added header, rephrased some comments

---
 run_tests.sh | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index 5e9fcb543d..0c51a14807 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,13 +1,24 @@
 #!/bin/bash
+#
+# This script gets invoked by the bot/test.sh script to run within the EESSI container
+# Thus, this script defines all of the steps that should run for the tests.
+# Note that, unless we have good reason, we don't run test steps in the prefix environment:
+# users also typically don't run in the prefix environment, and we want to check if the
+# software works well in that specific setup.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Caspar van Leeuwe (@casparvl)
+#
+# license: GPLv2
+#
+
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
 
-# Note: for these tests, we _don't_ run in the compat layer env
-# These tests should mimic what users do, and they are typically not in a prefix environment
-
-# Run eb --sanity-check-only on changed easyconfigs
-# TODO: in the future we may implement this as a light first check.
+# Git clone has to be run in compat layer, to make the git command available
+./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 
 # Run the test suite
-./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 ./test_suite.sh

From cdf79712184e89a31afaaff3880fc7cf0387b2e5 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:13:11 +0100
Subject: [PATCH 55/67] Fix headers

---
 run_tests.sh  |  2 +-
 test_suite.sh | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index 0c51a14807..de7fd8c2e1 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,7 +9,7 @@
 # This script is part of the EESSI software layer, see
 # https://github.com/EESSI/software-layer.git
 #
-# author: Caspar van Leeuwe (@casparvl)
+# author: Caspar van Leeuwen (@casparvl)
 #
 # license: GPLv2
 #
diff --git a/test_suite.sh b/test_suite.sh
index b3884c3b81..ffe8955f1e 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
 #
-# Run sanity check for all requested modules (and built dependencies)?
-# get ec from diff
-# set up EasyBuild
-# run `eb --sanity-only ec`
+# This script creates a ReFrame config file from a template, in which CPU properties get replaced
+# based on where this script is run (typically: a build node). Then, it runs the EESSI test suite.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Caspar van Leeuwen (@casparvl)
+#
+# license: GPLv2
 
 display_help() {
   echo "usage: $0 [OPTIONS]"

From 79a9ad320aec47ac368ed96f72e0f59a0cfbe9f5 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:16:10 +0100
Subject: [PATCH 56/67] Load default ReFrame version

---
 test_suite.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index ffe8955f1e..1f6d8ae44d 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -129,11 +129,11 @@ else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
-# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version
-# of the ReFrame module available in the current environment
-module load ReFrame/4.3.3
+# Load the ReFrame module
+# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+module load ReFrame
 if [[ $? -eq 0 ]]; then
-    echo_green ">> Loaded ReFrame/4.3.3"
+    echo_green ">> Loaded ReFrame module"
 else
     fatal_error "Failed to load the ReFrame module"
 fi

From 2f280e4304b520de295fdb408218e3125b153853 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:21:24 +0100
Subject: [PATCH 57/67] Point to docs for ReFrame config

---
 test_suite.sh | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1f6d8ae44d..d2403b91d8 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -130,7 +130,7 @@ else
 fi
 
 # Load the ReFrame module
-# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+ Currently, we load the default version. Maybe we should somehow make this configurable in the future?
 module load ReFrame
 if [[ $? -eq 0 ]]; then
     echo_green ">> Loaded ReFrame module"
@@ -139,15 +139,16 @@ else
 fi
 
 # Check ReFrame came with the hpctestlib and we can import it
-python3 -c 'import hpctestlib.sciapps.gromacs'
+reframe_import="hpctestlib.sciapps.gromacs"
+python3 -c "import ${reframe_import}"
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully found and imported hpctestlib.sciapps.gromas"
+    echo_green "Succesfully found and imported ${reframe_import}"
 else
-    fatal_error "Failed to load hpctestlib"
+    fatal_error "Failed to import ${reframe_import}"
 fi
 
-# Cloning should already be done by clone_eessi_test_suite.sh, which runs in compat layer to have 'git' available
-# git clone https://github.com/EESSI/test-suite EESSI-test-suite
+# Cloning should already be done in run_tests.sh before test_suite.sh is invoked
+# Check if that succeeded
 export TESTSUITEPREFIX=$PWD/EESSI-test-suite
 if [ -d $TESTSUITEPREFIX ]; then
     echo_green "Clone of the test suite $TESTSUITEPREFIX available, OK!"
@@ -157,14 +158,15 @@ fi
 export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
 
 # Check that we can import from the testsuite
-python3 -c 'import eessi.testsuite'
+testsuite_import="eessi.testsuite"
+python3 -c "import ${testsuite_import}"
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully found and imported eessi.testsuite"
+    echo_green "Succesfully found and imported ${testsuite_import}"
 else
-    fatal_error "Failed to import from eessi.testsuite in Python"
+    fatal_error "Failed to import ${testsuite_import}"
 fi
 
-# Configure ReFrame
+# Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration
 export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
 export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests

From e5d60b9c6bd2b3a39a32454a9ee6707320693c11 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:26:17 +0100
Subject: [PATCH 58/67] Fix silly mistake: I was overwriting the ReFrame config
 file with every sed call

---
 test_suite.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index d2403b91d8..755102e6bc 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -198,17 +198,18 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
 fi
-sed "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES}
+sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully ran reframe --version"
+    echo_green "Succesfully ran 'reframe --version'"
 else
-    fatal_error "Failed to run ReFrame --version"
+    fatal_error "Failed to run 'reframe --version'"
 fi
 
 # List the tests we want to run
@@ -224,7 +225,8 @@ fi
 # Run all tests
 echo "Running tests: reframe ${REFRAME_ARGS} --run"
 reframe ${REFRAME_ARGS} --run
-if [[ $? -eq 0 ]]; then
+reframe_exit_code=$?
+if [[ ${reframe_exit_code} -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
 else
     fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
@@ -233,4 +235,4 @@ fi
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
 
-exit 0
+exit ${reframe_exit_code}

From b5c60ccfc483602eb6cebb9e1f580c9d99cd964b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:28:32 +0100
Subject: [PATCH 59/67] See if @boegel is correct and if we don't actually need
 ANY software change to have the bot run...

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 4c73b5887a..82190071ab 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -15,4 +15,3 @@ easyconfigs:
       options:
         from-pr: 19573
   - scikit-learn-1.3.1-gfbf-2023a.eb
-  - patchelf-0.18.0-GCCcore-12.3.0.eb

From 69d6d2e227c6f2ceb0ad39068d6fe4a99d7fd24e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 14 Feb 2024 14:55:14 +0100
Subject: [PATCH 60/67] Add escape character to star, so that the regex pattern
 prints correctly on GitHub in the report

---
 bot/check-test.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index ef52206346..bc5ba84101 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -188,6 +188,8 @@ success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
 comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
+# Add an escape character to every *, for it to be printed correctly in the comment on GitHub
+GP_failed="${GP_failed//\*/\\*}"
 success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
 comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")

From 2dd048e96cef85b203b76a9ca330c5c72813dbbc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 15 Feb 2024 15:44:15 +0100
Subject: [PATCH 61/67] Add the list(SCALES.keys()) feature (though technically
 we could limit to singlenode already here...). Also, make sure color is not
 used for printing the ReFrame results, to avoid color-characters making it
 into the slurm output file

---
 reframe_config_bot.py.tmpl | 4 +++-
 test_suite.sh              | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl
index ad66a72041..b2f7916e79 100644
--- a/reframe_config_bot.py.tmpl
+++ b/reframe_config_bot.py.tmpl
@@ -19,7 +19,9 @@ site_configuration = {
                     'scheduler': 'local',
                     'launcher': 'mpirun',
                     'environs': ['default'],
-                    'features': [FEATURES[CPU]],
+                    'features': [
+                        FEATURES[CPU]
+                    ] + list(SCALES.keys()),
                     'processor': {
                         'num_cpus': __NUM_CPUS__,
                         'num_sockets': __NUM_SOCKETS__,
diff --git a/test_suite.sh b/test_suite.sh
index 755102e6bc..f8c46ef764 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -213,7 +213,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node'
+export REFRAME_ARGS='--tag CI --tag 1_node --nocolor'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From 0492b9e6d836ef75126ca5e8f613c7ad7487dc2b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 15 Feb 2024 15:45:25 +0100
Subject: [PATCH 62/67] Try the original regex patterns, capturing spaces more
 specifically, now that we got rid of color characters

---
 bot/check-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index bc5ba84101..4b5f7575e5 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -42,7 +42,7 @@ fi
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
-  GP_failed='\[.*FAILED.*\].*Ran .* test case'
+  GP_failed='\[\s*FAILED\s*\].*Ran .* test case'
   grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
@@ -65,7 +65,7 @@ fi
 SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
-  GP_success='\[.*PASSED.*\].*Ran .* test case'
+  GP_success='\[\s*PASSED\s*\].*Ran .* test case'
   grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file

From 8746a33de409e8509464c094100e28a0ef08f9ad Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 17 Feb 2024 01:53:55 +0100
Subject: [PATCH 63/67] fix minor bug + add debug output

---
 test_suite.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index f8c46ef764..b4d64c5850 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -130,7 +130,7 @@ else
 fi
 
 # Load the ReFrame module
- Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
 module load ReFrame
 if [[ $? -eq 0 ]]; then
     echo_green ">> Loaded ReFrame module"
@@ -138,6 +138,10 @@ else
     fatal_error "Failed to load the ReFrame module"
 fi
 
+which python3
+python3 -V
+python3 -c "import sys; print(sys.version_info)"
+
 # Check ReFrame came with the hpctestlib and we can import it
 reframe_import="hpctestlib.sciapps.gromacs"
 python3 -c "import ${reframe_import}"

From 2e23bf0984afed5cc1752ad3bb4b9332b3578f68 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 17 Feb 2024 02:09:42 +0100
Subject: [PATCH 64/67] list loaded modules and available Python modules

---
 test_suite.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test_suite.sh b/test_suite.sh
index b4d64c5850..ba4975e2cb 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -138,9 +138,11 @@ else
     fatal_error "Failed to load the ReFrame module"
 fi
 
+module list
 which python3
 python3 -V
 python3 -c "import sys; print(sys.version_info)"
+module avail Python/
 
 # Check ReFrame came with the hpctestlib and we can import it
 reframe_import="hpctestlib.sciapps.gromacs"

From d56a9cd2399a60a2e1d4485bdf3032b3c2ad42b9 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 17 Feb 2024 02:16:35 +0100
Subject: [PATCH 65/67] run test suite in prefix env

---
 run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run_tests.sh b/run_tests.sh
index de7fd8c2e1..677f078593 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -21,4 +21,4 @@ source ${base_dir}/init/eessi_defaults
 ./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 
 # Run the test suite
-./test_suite.sh
+./run_in_compat_layer_env.sh ./test_suite.sh

From 359a4571d3e424a13c0fb9ff3704f55d8a0a385b Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 17 Feb 2024 21:50:49 +0100
Subject: [PATCH 66/67] source init script instead of running in compat layer

---
 run_tests.sh  |   4 +-
 test_suite.sh | 110 ++++++++++++++++++++++++++------------------------
 2 files changed, 60 insertions(+), 54 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index 677f078593..b622af4be3 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -21,4 +21,6 @@ source ${base_dir}/init/eessi_defaults
 ./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 
 # Run the test suite
-./run_in_compat_layer_env.sh ./test_suite.sh
+#./run_in_compat_layer_env.sh ./test_suite.sh
+# source init script instead of running in compat layer
+./test_suite.sh
diff --git a/test_suite.sh b/test_suite.sh
index ba4975e2cb..87c52d3b64 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -74,60 +74,64 @@ fi
 TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
-
-source $TOPDIR/init/minimal_eessi_env
-
-if [ -d $EESSI_CVMFS_REPO ]; then
-    echo_green "$EESSI_CVMFS_REPO available, OK!"
-else
-    fatal_error "$EESSI_CVMFS_REPO is not available!"
-fi
-
-# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
-export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
-
-echo ">> Determining software subdirectory to use for current build/test host..."
-if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
-else
-  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
-fi
-
-# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
-# $EESSI_SILENT - don't print any messages
-# $EESSI_BASIC_ENV - give a basic set of environment variables
-EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
-
-if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
-    fatal_error "Failed to determine software subdirectory?!"
-elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
-    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
-else
-    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
-fi
-
-echo ">> Initializing Lmod..."
-source $EPREFIX/usr/share/Lmod/init/bash
-ml_version_out=$TMPDIR/ml.out
-ml --version &> $ml_version_out
-if [[ $? -eq 0 ]]; then
-    echo_green ">> Found Lmod ${LMOD_VERSION}"
-else
-    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
-fi
-
-echo ">> Setting up \$MODULEPATH..."
-# make sure no modules are loaded
+####
+####source $TOPDIR/init/minimal_eessi_env
+####
+####if [ -d $EESSI_CVMFS_REPO ]; then
+####    echo_green "$EESSI_CVMFS_REPO available, OK!"
+####else
+####    fatal_error "$EESSI_CVMFS_REPO is not available!"
+####fi
+####
+##### avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
+####export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+####
+####echo ">> Determining software subdirectory to use for current build/test host..."
+####if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+####  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+####  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+####else
+####  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+####fi
+####
+##### Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+##### $EESSI_SILENT - don't print any messages
+##### $EESSI_BASIC_ENV - give a basic set of environment variables
+####EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+####
+####if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
+####    fatal_error "Failed to determine software subdirectory?!"
+####elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+####    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
+####else
+####    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
+####fi
+####
+####echo ">> Initializing Lmod..."
+####source $EPREFIX/usr/share/Lmod/init/bash
+####ml_version_out=$TMPDIR/ml.out
+####ml --version &> $ml_version_out
+####if [[ $? -eq 0 ]]; then
+####    echo_green ">> Found Lmod ${LMOD_VERSION}"
+####else
+####    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
+####fi
+####
+####echo ">> Setting up \$MODULEPATH..."
+##### make sure no modules are loaded
+####module --force purge
+##### ignore current $MODULEPATH entirely
+####module unuse $MODULEPATH
+####module use ${EESSI_SOFTWARE_PATH}/modules/all
+####if [[ -z ${MODULEPATH} ]]; then
+####    fatal_error "Failed to set up \$MODULEPATH?!"
+####else
+####    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
+####fi
+####
 module --force purge
-# ignore current $MODULEPATH entirely
-module unuse $MODULEPATH
-module use ${EESSI_SOFTWARE_PATH}/modules/all
-if [[ -z ${MODULEPATH} ]]; then
-    fatal_error "Failed to set up \$MODULEPATH?!"
-else
-    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
-fi
+source $TOPDIR/init/minimal_eessi_env
+source /cvmfs/${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
 
 # Load the ReFrame module
 # Currently, we load the default version. Maybe we should somehow make this configurable in the future?

From 642a41646df20975635fcc9bbd6f524f26136eea Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 17 Feb 2024 22:13:00 +0100
Subject: [PATCH 67/67] EESSI_CVMFS_REPO already contains '/cvmfs'

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 87c52d3b64..2977413205 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -131,7 +131,7 @@ echo ">> Setting up environment..."
 ####
 module --force purge
 source $TOPDIR/init/minimal_eessi_env
-source /cvmfs/${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
+source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
 
 # Load the ReFrame module
 # Currently, we load the default version. Maybe we should somehow make this configurable in the future?